diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 0e428c511..60d9c0da3 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -33,7 +33,7 @@ ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 ENV ARKIME_DIR "/opt/arkime" -ENV ARKIME_VERSION "5.4.0" +ENV ARKIME_VERSION "5.5.0" ENV ARKIME_DEB_URL "https://github.com/arkime/arkime/releases/download/v${ARKIME_VERSION}/arkime_${ARKIME_VERSION}-1.debian12_XXX.deb" ENV ARKIME_JA4_SO_URL "https://github.com/arkime/arkime/releases/download/v${ARKIME_VERSION}/ja4plus.XXX.so" ENV ARKIME_LOCALELASTICSEARCH no @@ -149,7 +149,7 @@ RUN export DEBARCH=$(dpkg --print-architecture) && \ mkdir -p "${ARKIME_DIR}"/plugins && \ curl -fsSL -o "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" "$(echo "${ARKIME_JA4_SO_URL}" | sed "s/XXX/${DEBARCH}/g")" && \ chmod 755 "${ARKIME_DIR}/plugins/ja4plus.${DEBARCH}.so" && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==5.0.3 && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir beautifulsoup4 pyzmq watchdog==6.0.0 && \ ln -sfr $ARKIME_DIR/bin/npm /usr/local/bin/npm && \ ln -sfr $ARKIME_DIR/bin/node /usr/local/bin/node && \ ln -sfr $ARKIME_DIR/bin/npx /usr/local/bin/npx && \ diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile index 6a82cfe9f..3bfb5f610 100644 --- a/Dockerfiles/dashboards.Dockerfile +++ b/Dockerfiles/dashboards.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch-dashboards:2.17.1 +FROM opensearchproject/opensearch-dashboards:2.18.0 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' @@ -42,10 +42,10 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') # Malcolm manages authentication and encryption via NGINX reverse proxy /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \ cd /tmp && \ - # unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ - # sed -i "s/2\.16\.0/2\.17\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ - # sed -i "s/2\.16\.0/2\.17\.0/g" opensearch-dashboards/transformVis/package.json && \ - # zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + unzip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ + sed -i "s/2\.17\.1/2\.18\.0/g" opensearch-dashboards/transformVis/opensearch_dashboards.json && \ + sed -i "s/2\.17\.1/2\.18\.0/g" opensearch-dashboards/transformVis/package.json && \ + zip transformVis.zip opensearch-dashboards/transformVis/opensearch_dashboards.json opensearch-dashboards/transformVis/package.json && \ cd /usr/share/opensearch-dashboards/plugins && \ /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/transformVis.zip --allow-root && \ rm -rf /tmp/transformVis /tmp/opensearch-dashboards && \ diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile index af9277a62..03cf7dd21 100644 --- a/Dockerfiles/file-monitor.Dockerfile +++ b/Dockerfiles/file-monitor.Dockerfile @@ -159,7 +159,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') python-magic \ stream-zip \ supervisor \ - watchdog==5.0.3 \ + watchdog==6.0.0 \ yara-python && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile index 9f496c004..18932f286 100644 --- a/Dockerfiles/filebeat.Dockerfile +++ b/Dockerfiles/filebeat.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/beats/filebeat-oss:8.15.3 +FROM docker.elastic.co/beats/filebeat-oss:8.16.0 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" @@ -74,7 +74,7 @@ ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV YQ_VERSION "4.44.3" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_" -ENV EVTX_VERSION "0.8.3" +ENV EVTX_VERSION "0.8.4" ENV EVTX_URL "https://github.com/omerbenamram/evtx/releases/download/v${EVTX_VERSION}/evtx_dump-v${EVTX_VERSION}-XXX-unknown-linux-gnu" USER root @@ -100,15 +100,14 @@ RUN export EVTXARCH=$(uname -m | sed 's/arm64/aarch64/') && \ psmisc \ python3-pip \ python3-setuptools \ - python3.9 \ + python3 \ rsync \ tar \ tini \ unar \ unzip \ xz-utils && \ - ln -s -f -r /usr/bin/python3.9 /usr/bin/python3 && \ - python3.9 -m pip install --no-compile --no-cache-dir patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==5.0.3 && \ + python3 -m pip install --no-compile --no-cache-dir --break-system-packages patool entrypoint2 pyunpack python-magic ordered-set supervisor watchdog==6.0.0 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/local/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile index 69348bb05..c2ef0f626 100644 --- a/Dockerfiles/logstash.Dockerfile +++ b/Dockerfiles/logstash.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.elastic.co/logstash/logstash-oss:8.15.3 +FROM docker.elastic.co/logstash/logstash-oss:8.16.0 LABEL maintainer="malcolm@inl.gov" LABEL org.opencontainers.image.authors='malcolm@inl.gov' diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile index 9bf794a4f..c422a0ce3 100644 --- a/Dockerfiles/netbox.Dockerfile +++ b/Dockerfiles/netbox.Dockerfile @@ -31,6 +31,7 @@ ENV SUPERCRONIC_CRONTAB "/etc/crontab" ENV NETBOX_INITIALIZERS_VERSION "50d077d" ENV NETBOX_TOPOLOGY_VERSION "4.0.1" +ENV NETBOX_HEALTHCHECK_VERSION "0.2.0" ENV YQ_VERSION "4.44.3" ENV YQ_URL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_" @@ -86,6 +87,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') "${NETBOX_PATH}/venv/bin/python" -m pip install --break-system-packages --no-compile --no-cache-dir \ "git+https://github.com/tobiasge/netbox-initializers@${NETBOX_INITIALIZERS_VERSION}" \ "git+https://github.com/netbox-community/netbox-topology-views@v${NETBOX_TOPOLOGY_VERSION}" \ + "git+https://github.com/netbox-community/netbox-healthcheck-plugin@v${NETBOX_HEALTHCHECK_VERSION}" \ psycopg2 \ pynetbox \ python-magic \ diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile index 2daa274a5..84264336d 100644 --- a/Dockerfiles/opensearch.Dockerfile +++ b/Dockerfiles/opensearch.Dockerfile @@ -1,4 +1,4 @@ -FROM opensearchproject/opensearch:2.17.1 +FROM opensearchproject/opensearch:2.18.0 # Copyright (c) 2024 Battelle Energy Alliance, LLC. All rights reserved. LABEL maintainer="malcolm@inl.gov" diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile index 6b62df402..f1c83c8da 100644 --- a/Dockerfiles/pcap-monitor.Dockerfile +++ b/Dockerfiles/pcap-monitor.Dockerfile @@ -68,7 +68,7 @@ RUN apt-get -q update && \ python-magic \ pyzmq \ requests \ - watchdog==5.0.3 && \ + watchdog==6.0.0 && \ groupadd --gid ${DEFAULT_GID} ${PGROUP} && \ useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER} diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile index 2241a50d3..166d3be9d 100644 --- a/Dockerfiles/suricata.Dockerfile +++ b/Dockerfiles/suricata.Dockerfile @@ -108,7 +108,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') apt-get install -q -y --no-install-recommends -t bookworm-backports \ suricata=${SURICATA_VERSION_PATTERN} \ suricata-update && \ - python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==5.0.3 && \ + python3 -m pip install --break-system-packages --no-compile --no-cache-dir watchdog==6.0.0 && \ curl -fsSL -o /usr/local/bin/supercronic "${SUPERCRONIC_URL}${BINARCH}" && \ chmod +x /usr/local/bin/supercronic && \ curl -fsSL -o /usr/bin/yq "${YQ_URL}${BINARCH}" && \ diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile index a823a1603..d3253806c 100644 --- a/Dockerfiles/zeek.Dockerfile +++ b/Dockerfiles/zeek.Dockerfile @@ -87,6 +87,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') libmaxminddb0 \ libpcap-dev \ libpcap0.8 \ + librdkafka-dev \ libssl-dev \ libssl3 \ libtcmalloc-minimal4 \ @@ -107,6 +108,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') python3-setuptools \ python3-tz \ python3-wheel \ + python3-yaml \ python3-zmq \ rsync \ supervisor \ @@ -115,7 +117,12 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') vim-tiny \ xxd \ zlib1g-dev && \ - python3 -m pip install --break-system-packages --no-cache-dir pymisp stix2 taxii2-client dateparser && \ + python3 -m pip install --break-system-packages --no-cache-dir \ + dateparser \ + git+https://github.com/google/mandiant-ti-client \ + pymisp \ + stix2 \ + taxii2-client && \ mkdir -p /tmp/zeek-packages && \ bash /usr/local/bin/zeek-deb-download.sh -o /tmp/zeek-packages -z "${ZEEK_VERSION}" && \ dpkg -i /tmp/zeek-packages/*.deb && \ @@ -133,6 +140,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') ( find "${ZEEK_DIR}"/lib/zeek/plugins/packages -type f -name "*.hlto" -exec chmod 755 "{}" \; || true ) && \ mkdir -p "${ZEEK_DIR}"/share/zeek/site/intel/STIX && \ mkdir -p "${ZEEK_DIR}"/share/zeek/site/intel/MISP && \ + mkdir -p "${ZEEK_DIR}"/share/zeek/site/intel/Mandiant && \ mkdir -p "${ZEEK_DIR}"/share/zeek/site/custom && \ touch "${ZEEK_DIR}"/share/zeek/site/intel/__load__.zeek && \ touch "${ZEEK_DIR}"/share/zeek/site/custom/__load__.zeek && \ @@ -174,7 +182,7 @@ RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \ # sanity checks to make sure the plugins installed and copied over correctly # these ENVs should match the third party scripts/plugins installed by zeek_install_plugins.sh -ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_HART_IP_UDP|ANALYZER_SPICY_HART_IP_TCP|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_SPICY_GE_SRTP|ANALYZER_SPICY_PROFINET_IO_CM|ANALYZER_S7COMM_TCP|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)" +ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_HART_IP_UDP|ANALYZER_SPICY_HART_IP_TCP|ANALYZER_SYNCHROPHASOR_TCP|ANALYZER_GENISYS_TCP|ANALYZER_SPICY_GE_SRTP|ANALYZER_SPICY_PROFINET_IO_CM|ANALYZER_S7COMM_TCP|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS|Seiso::Kafka)" ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bro-is-darknet/main|bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja4/main|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)" RUN mkdir -p /tmp/logs && \ @@ -183,7 +191,7 @@ RUN mkdir -p /tmp/logs && \ export ZEEK_THIRD_PARTY_SCRIPTS_COUNT=$(echo "$ZEEK_THIRD_PARTY_SCRIPTS_GREP" | grep -P -o "\([^)]+\)" | head -n 1 | sed "s/^(//" | sed "s/)$//" | tr '|' '\n' | wc -l) && \ "$ZEEK_DIR"/bin/zeek-offline -NN local >zeeknn.log 2>/dev/null && \ bash -c "(( $(grep -cP "$ZEEK_THIRD_PARTY_PLUGINS_GREP" zeeknn.log) >= $ZEEK_THIRD_PARTY_PLUGINS_COUNT)) && echo $ZEEK_THIRD_PARTY_PLUGINS_COUNT' Zeek plugins loaded correctly' || (echo 'One or more Zeek plugins did not load correctly' && cat zeeknn.log && exit 1)" && \ - "$ZEEK_DIR"/bin/zeek-offline -C -r /tmp/pcaps/udp.pcap local policy/misc/loaded-scripts 2>/dev/null && \ + "$ZEEK_DIR"/bin/zeek-offline -C -r /tmp/pcaps/udp.pcap local policy/misc/loaded-scripts >loaded_scripts.log 2>/dev/null && \ bash -c "(( $(grep -cP "$ZEEK_THIRD_PARTY_SCRIPTS_GREP" loaded_scripts.log) == $ZEEK_THIRD_PARTY_SCRIPTS_COUNT)) && echo $ZEEK_THIRD_PARTY_SCRIPTS_COUNT' Zeek scripts loaded correctly' || (echo 'One or more Zeek scripts did not load correctly' && cat loaded_scripts.log && exit 1)" && \ cd /tmp && \ rm -rf /tmp/logs /tmp/pcaps @@ -195,8 +203,9 @@ ARG ZEEK_PCAP_PROCESSOR=true #Whether or not to run "zeek -r XXXXX.pcap local" on each pcap file ARG ZEEK_AUTO_ANALYZE_PCAP_FILES=false ARG ZEEK_AUTO_ANALYZE_PCAP_THREADS=1 -#Whether or not to refresh intel at various points during processing -ARG ZEEK_INTEL_REFRESH_ON_ENTRYPOINT=false +#Whether or not to do first intel refresh under supervisord +ARG ZEEK_INTEL_REFRESH_ON_STARTUP=false +#Whether or not to do first intel refresh under zeekdeploy.sh ARG ZEEK_INTEL_REFRESH_ON_DEPLOY=false ARG ZEEK_INTEL_REFRESH_CRON_EXPRESSION= ARG ZEEK_INTEL_ITEM_EXPIRATION=-1min @@ -219,7 +228,7 @@ ARG PCAP_NODE_NAME=malcolm ENV AUTO_TAG $AUTO_TAG ENV ZEEK_PCAP_PROCESSOR $ZEEK_PCAP_PROCESSOR -ENV ZEEK_INTEL_REFRESH_ON_ENTRYPOINT $ZEEK_INTEL_REFRESH_ON_ENTRYPOINT +ENV ZEEK_INTEL_REFRESH_ON_STARTUP $ZEEK_INTEL_REFRESH_ON_STARTUP ENV ZEEK_INTEL_REFRESH_ON_DEPLOY $ZEEK_INTEL_REFRESH_ON_DEPLOY ENV ZEEK_INTEL_REFRESH_CRON_EXPRESSION $ZEEK_INTEL_REFRESH_CRON_EXPRESSION ENV ZEEK_AUTO_ANALYZE_PCAP_FILES $ZEEK_AUTO_ANALYZE_PCAP_FILES @@ -249,8 +258,6 @@ ARG ZEEK_DISABLE_SSL_VALIDATE_CERTS= ARG ZEEK_DISABLE_TRACK_ALL_ASSETS= ARG ZEEK_DISABLE_DETECT_ROUTERS=true ARG ZEEK_DISABLE_BEST_GUESS_ICS=true -# TODO: assess spicy-analyzer that replace built-in Zeek parsers -# for now, disable them by default when a Zeek parser exists ARG ZEEK_DISABLE_SPICY_IPSEC= ARG ZEEK_DISABLE_SPICY_LDAP= ARG ZEEK_DISABLE_SPICY_OPENVPN= @@ -260,6 +267,9 @@ ARG ZEEK_DISABLE_SPICY_TAILSCALE= ARG ZEEK_DISABLE_SPICY_TFTP= ARG ZEEK_DISABLE_SPICY_WIREGUARD= ARG ZEEK_SYNCHROPHASOR_DETAILED= +ARG ZEEK_KAFKA_ENABLED= +ARG ZEEK_KAFKA_BROKERS=kafka.local:9091 +ARG ZEEK_KAFKA_TOPIC=zeek ENV ZEEK_DISABLE_STATS $ZEEK_DISABLE_STATS ENV ZEEK_DISABLE_HASH_ALL_FILES $ZEEK_DISABLE_HASH_ALL_FILES @@ -278,6 +288,9 @@ ENV ZEEK_DISABLE_SPICY_TAILSCALE $ZEEK_DISABLE_SPICY_TAILSCALE ENV ZEEK_DISABLE_SPICY_TFTP $ZEEK_DISABLE_SPICY_TFTP ENV ZEEK_DISABLE_SPICY_WIREGUARD $ZEEK_DISABLE_SPICY_WIREGUARD ENV ZEEK_SYNCHROPHASOR_DETAILED $ZEEK_SYNCHROPHASOR_DETAILED +ENV ZEEK_KAFKA_ENABLED $ZEEK_KAFKA_ENABLED +ENV ZEEK_KAFKA_BROKERS $ZEEK_KAFKA_BROKERS +ENV ZEEK_KAFKA_TOPIC $ZEEK_KAFKA_TOPIC # This is in part to handle an issue when running with rootless podman and # "userns_mode: keep-id". It seems that anything defined as a VOLUME diff --git a/api/project/__init__.py b/api/project/__init__.py index 004cc2cf2..3e1d45f65 100644 --- a/api/project/__init__.py +++ b/api/project/__init__.py @@ -15,7 +15,7 @@ from collections import defaultdict, OrderedDict from collections.abc import Iterable -from datetime import datetime +from datetime import datetime, timezone from flask import Flask, jsonify, request from requests.auth import HTTPBasicAuth from urllib.parse import urlparse @@ -169,15 +169,6 @@ missing_field_map['ip'] = '0.0.0.0' missing_field_map['long'] = 0 -logstash_default_pipelines = [ - "malcolm-beats", - "malcolm-enrichment", - "malcolm-input", - "malcolm-output", - "malcolm-suricata", - "malcolm-zeek", -] - urllib3.disable_warnings() warnings.filterwarnings( "ignore", @@ -239,14 +230,14 @@ if databaseMode == malcolm_utils.DatabaseMode.ElasticsearchRemote: import elasticsearch as DatabaseImport - from elasticsearch_dsl import Search as SearchClass + from elasticsearch_dsl import Search as SearchClass, A as AggregationClass DatabaseClass = DatabaseImport.Elasticsearch if opensearchHttpAuth: DatabaseInitArgs['basic_auth'] = opensearchHttpAuth else: import opensearchpy as DatabaseImport - from opensearchpy import Search as SearchClass + from opensearchpy import Search as SearchClass, A as AggregationClass DatabaseClass = DatabaseImport.OpenSearch if opensearchHttpAuth: @@ -937,7 +928,7 @@ def ready(): logstash_lumberjack true or false, the ready status of Logstash's lumberjack protocol listener logstash_pipelines - true or false, the ready status of Logstash's default pipelines + true or false, the ready status of Logstash's pipelines netbox true or false, the ready status of NetBox opensearch @@ -998,9 +989,9 @@ def ready(): print(f"{type(e).__name__}: {str(e)} getting freq status") try: - logstashStats = requests.get(f'{logstashUrl}/_node').json() + logstashHealth = requests.get(f'{logstashUrl}/_health_report').json() except Exception as e: - logstashStats = {} + logstashHealth = {} if debugApi: print(f"{type(e).__name__}: {str(e)} getting Logstash node status") @@ -1012,7 +1003,7 @@ def ready(): print(f"{type(e).__name__}: {str(e)} getting Logstash lumberjack listener status") try: - netboxStatus = requests.get(f'{netboxUrl}/api/status').json() + netboxStatus = requests.get(f'{netboxUrl}/plugins/netbox_healthcheck_plugin/healthcheck/?format=json').json() except Exception as e: netboxStatus = {} if debugApi: @@ -1057,12 +1048,13 @@ def ready(): filebeat_tcp=filebeatTcpJsonStatus, freq=freqStatus, logstash_lumberjack=logstashLJStatus, - logstash_pipelines=(malcolm_utils.deep_get(logstashStats, ["status"]) == "green") - and all( - pipeline in malcolm_utils.deep_get(logstashStats, ["pipelines"], {}) - for pipeline in logstash_default_pipelines + logstash_pipelines=(malcolm_utils.deep_get(logstashHealth, ["status"]) == "green") + and (malcolm_utils.deep_get(logstashHealth, ["indicators", "pipelines", "status"]) == "green"), + netbox=bool( + isinstance(netboxStatus, dict) + and netboxStatus + and all(value == "working" for value in netboxStatus.values()) ), - netbox=bool(malcolm_utils.deep_get(netboxStatus, ["netbox-version"])), opensearch=(malcolm_utils.deep_get(openSearchHealth, ["status"], 'red') != "red"), pcap_monitor=pcapMonitorStatus, zeek_extracted_file_logger=zeekExtractedFileLoggerStatus, @@ -1070,6 +1062,143 @@ def ready(): ) +@app.route( + f"{('/' + app.config['MALCOLM_API_PREFIX']) if app.config['MALCOLM_API_PREFIX'] else ''}/dashboard-export/", + methods=['GET', 'POST'], +) +def dashboard_export(dashid): + """Uses the opensearch dashboards API to export a dashboard. Also handles the _REPLACER strings + as described in "Adding new visualizations and dashboards" at + https://idaholab.github.io/Malcolm/docs/contributing-dashboards.html#DashboardsNewViz + + Parameters + ---------- + dashid : string + the ID of the dashboard to export + request : Request + Uses 'replace' from requests arguments, true (default) or false; indicates whether or not to do + MALCOLM_NETWORK_INDEX_PATTERN_REPLACER, MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER, + MALCOLM_OTHER_INDEX_PATTERN_REPLACER + + Returns + ------- + content + The JSON of the exported dashboard + """ + + args = get_request_arguments(request) + try: + # call the API to get the dashboard JSON + response = requests.get( + f"{dashboardsUrl}/api/{'kibana' if (databaseMode == malcolm_utils.DatabaseMode.ElasticsearchRemote) else 'opensearch-dashboards'}/dashboards/export", + params={ + 'dashboard': dashid, + }, + auth=opensearchReqHttpAuth, + verify=opensearchSslVerify, + ) + response.raise_for_status() + + if doReplacers := malcolm_utils.str2bool(args.get('replace', 'true')): + # replace references to index pattern names with the _REPLACER strings, which will allow other Malcolm + # instances that use different index pattern names to import them and substitute their own names + replacements = { + app.config['MALCOLM_NETWORK_INDEX_PATTERN']: 'MALCOLM_NETWORK_INDEX_PATTERN_REPLACER', + app.config['MALCOLM_NETWORK_INDEX_TIME_FIELD']: 'MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER', + app.config['MALCOLM_OTHER_INDEX_PATTERN']: 'MALCOLM_OTHER_INDEX_PATTERN_REPLACER', + } + pattern = re.compile('|'.join(re.escape(key) for key in replacements)) + responseText = pattern.sub(lambda match: replacements[match.group(0)], response.text) + else: + # ... or just return it as-is + responseText = response.text + + # remove index pattern definition from exported dashboard as they get created programatically + # on Malcolm startup and we don't want them to come in with imported dashboards + if responseParsed := malcolm_utils.LoadStrIfJson(responseText): + if 'objects' in responseParsed and isinstance(responseParsed['objects'], list): + responseParsed['objects'] = [ + o + for o in responseParsed['objects'] + if not ( + (o.get("type") == "index-pattern") + and ( + o.get("id") + in [ + ( + "MALCOLM_NETWORK_INDEX_PATTERN_REPLACER" + if doReplacers + else app.config['MALCOLM_NETWORK_INDEX_PATTERN'] + ), + ( + "MALCOLM_OTHER_INDEX_PATTERN_REPLACER" + if doReplacers + else app.config['MALCOLM_OTHER_INDEX_PATTERN'] + ), + ] + ) + ) + ] + return jsonify(responseParsed) + + else: + # what we got back from the API wasn't valid JSON, so sad + return jsonify(error=f'Could not process export response for {dashid}') + + except Exception as e: + errStr = f"{type(e).__name__}: {str(e)} exporting OpenSearch Dashboard {dashid}" + if debugApi: + print(errStr) + return jsonify(error=errStr) + + +@app.route( + f"{('/' + app.config['MALCOLM_API_PREFIX']) if app.config['MALCOLM_API_PREFIX'] else ''}/ingest-stats", + methods=['GET'], +) +def ingest_stats(): + """Provide an aggregation of each log source (host.name) with it's latest event.ingested + time. This can be used to know the most recent time a document was written from each + network sensor. + + Parameters + ---------- + request : Request + Uses 'doctype' from request arguments + Returns + ------- + fields + A dict where key is host.name and value is max(event.ingested) for that host + """ + global databaseClient + global SearchClass + global AggregationClass + + result = {} + try: + s = SearchClass( + using=databaseClient, + index=index_from_args(get_request_arguments(request)), + ).extra(size=0) + + hostAgg = AggregationClass('terms', field='host.name') + maxIngestAgg = AggregationClass('max', field='event.ingested') + s.aggs.bucket('host_names', hostAgg).metric('max_event_ingested', maxIngestAgg) + response = s.execute() + + result = { + bucket.key: datetime.fromtimestamp(bucket.max_event_ingested.value / 1000, timezone.utc) + .replace(microsecond=0) + .isoformat() + for bucket in response.aggregations.host_names.buckets + } + except Exception as e: + if debugApi: + print(f"{type(e).__name__}: \"{str(e)}\" getting ingest stats") + + return jsonify(result) + + @app.route( f"{('/' + app.config['MALCOLM_API_PREFIX']) if app.config['MALCOLM_API_PREFIX'] else ''}/ping", methods=['GET'] ) diff --git a/api/requirements.txt b/api/requirements.txt index aa497c6ab..3a87cbcf6 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -5,6 +5,6 @@ opensearch-py==2.6.0 requests==2.32.0 regex==2022.3.2 dateparser==1.1.1 -elasticsearch==8.15.1 -elasticsearch-dsl==8.15.4 +elasticsearch==8.16.0 +elasticsearch-dsl==8.16.0 psutil==5.9.8 \ No newline at end of file diff --git a/arkime/etc/config.ini b/arkime/etc/config.ini index 78c93ac50..4aabd2f3a 100644 --- a/arkime/etc/config.ini +++ b/arkime/etc/config.ini @@ -1237,19 +1237,11 @@ zeek.http.server_header_names=db:zeek.http.server_header_names;group:zeek_http;k # intel.log # https://docs.zeek.org/en/stable/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Info -zeek.intel.seen_indicator=db:zeek.intel.seen_indicator;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Indicator;help:Indicator -zeek.intel.seen_indicator_type=db:zeek.intel.seen_indicator_type;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Indicator Type;help:Indicator Type zeek.intel.seen_where=db:zeek.intel.seen_where;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Where Discovered;help:Where Discovered zeek.intel.seen_node=db:zeek.intel.seen_node;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Discovered Node;help:Discovered Node -zeek.intel.matched=db:zeek.intel.matched;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Match Indicator;help:Match Indicator -zeek.intel.sources=db:zeek.intel.sources;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:Match Source;help:Match Source zeek.intel.file_mime_type=db:zeek.intel.file_mime_type;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:File Magic;help:File Magic zeek.intel.file_description=db:zeek.intel.file_description;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:File Description;help:File Description -zeek.intel.cif_tags=db:zeek.intel.cif_tags;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:CIF Tags;help:CIF Tags zeek.intel.cif_confidence=db:zeek.intel.cif_confidence;group:zeek_intel;kind:float;viewerOnly:true;friendly:CIF Confidence;help:CIF Confidence -zeek.intel.cif_source=db:zeek.intel.cif_source;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:CIF Source;help:CIF Source -zeek.intel.cif_description=db:zeek.intel.cif_description;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:CIF Description;help:CIF Description -zeek.intel.cif_firstseen=db:zeek.intel.cif_firstseen;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:CIF First Seen;help:CIF First Seen zeek.intel.cif_lastseen=db:zeek.intel.cif_lastseen;group:zeek_intel;kind:termfield;viewerOnly:true;friendly:CIF Last Seen;help:CIF Last Seen # ipsec.log @@ -2073,6 +2065,23 @@ zeek.opcua_binary_variant_metadata.built_in_data_type=db:zeek.opcua_binary_varia zeek.opcua_binary_variant_metadata.built_in_data_type_str=db:zeek.opcua_binary_variant_metadata.built_in_data_type_str;group:zeek_opcua;kind:termfield;viewerOnly:true;friendly:built_in_data_type_str;help:built_in_data_type_str zeek.opcua_binary_variant_metadata.variant_data_array_dim=db:zeek.opcua_binary_variant_metadata.variant_data_array_dim;group:zeek_opcua;kind:integer;viewerOnly:true;friendly:variant_data_array_dim;help:variant_data_array_dim +# opcua_binary_write.log +# https://github.com/cisagov/icsnpp-opcua-binary +zeek.opcua_binary_write.node_id_encoding_mask=db:zeek.opcua_binary_write.node_id_encoding_mask;group:zeek_opcua_binary_write;kind:termfield;friendly:node_id_encoding_mask;help:node_id_encoding_mask +zeek.opcua_binary_write.node_id_namespace_idx=db:zeek.opcua_binary_write.node_id_namespace_idx;group:zeek_opcua_binary_write;kind:integer;friendly:node_id_namespace_idx;help:node_id_namespace_idx +zeek.opcua_binary_write.node_id_numeric=db:zeek.opcua_binary_write.node_id_numeric;group:zeek_opcua_binary_write;kind:integer;friendly:node_id_numeric;help:node_id_numeric +zeek.opcua_binary_write.node_id_string=db:zeek.opcua_binary_write.node_id_string;group:zeek_opcua_binary_write;kind:termfield;friendly:node_id_string;help:node_id_string +zeek.opcua_binary_write.node_id_guid=db:zeek.opcua_binary_write.node_id_guid;group:zeek_opcua_binary_write;kind:termfield;friendly:node_id_guid;help:node_id_guid +zeek.opcua_binary_write.node_id_opaque=db:zeek.opcua_binary_write.node_id_opaque;group:zeek_opcua_binary_write;kind:termfield;friendly:node_id_opaque;help:node_id_opaque +zeek.opcua_binary_write.attribute_id=db:zeek.opcua_binary_write.attribute_id;group:zeek_opcua_binary_write;kind:integer;friendly:attribute_id;help:attribute_id +zeek.opcua_binary_write.attribute_id_str=db:zeek.opcua_binary_write.attribute_id_str;group:zeek_opcua_binary_write;kind:termfield;friendly:attribute_id_str;help:attribute_id_str +zeek.opcua_binary_write.index_range=db:zeek.opcua_binary_write.index_range;group:zeek_opcua_binary_write;kind:termfield;friendly:index_range;help:index_range +zeek.opcua_binary_write.data_value_encoding_mask=db:zeek.opcua_binary_write.data_value_encoding_mask;group:zeek_opcua_binary_write;kind:termfield;friendly:data_value_encoding_mask;help:data_value_encoding_mask +zeek.opcua_binary_write.source_timestamp=db:zeek.opcua_binary_write.source_timestamp;group:zeek_opcua_binary_write;kind:termfield;friendly:source_timestamp;help:source_timestamp +zeek.opcua_binary_write.source_pico_sec=db:zeek.opcua_binary_write.source_pico_sec;group:zeek_opcua_binary_write;kind:integer;friendly:source_pico_sec;help:source_pico_sec +zeek.opcua_binary_write.server_timestamp=db:zeek.opcua_binary_write.server_timestamp;group:zeek_opcua_binary_write;kind:termfield;friendly:server_timestamp;help:server_timestamp +zeek.opcua_binary_write.server_pico_sec=db:zeek.opcua_binary_write.server_pico_sec;group:zeek_opcua_binary_write;kind:integer;friendly:server_pico_sec;help:server_pico_sec + # ocsp.log # https://docs.zeek.org/en/stable/scripts/policy/files/x509/log-ocsp.zeek.html#type-OCSP::Info zeek.ocsp.hashAlgorithm=db:zeek.ocsp.hashAlgorithm;group:zeek_ocsp;kind:termfield;viewerOnly:true;friendly:Issuer Name and Key Hash Algorithm;help:Issuer Name and Key Hash Algorithm @@ -3344,7 +3353,7 @@ o_zeek_hart_ip=require:zeek.hart_ip;title:Zeek hart_ip.log;fields:zeek.hart_ip.c o_zeek_hart_ip_session_record=require:zeek.hart_ip_session_record;title:Zeek hart_ip_session_record.log;fields:zeek.hart_ip.session_log_record_link_id,zeek.hart_ip_session_record.session_log_record_client_i_pv4_address,zeek.hart_ip_session_record.session_log_record_client_i_pv6_address,zeek.hart_ip_session_record.session_log_record_client_port,zeek.hart_ip_session_record.session_log_record_server_port,zeek.hart_ip_session_record.session_log_record_connect_time,zeek.hart_ip_session_record.session_log_record_disconnect_time,zeek.hart_ip_session_record.session_log_record_session_status_summary_undefined_bits,zeek.hart_ip_session_record.session_log_record_session_status_summary_insecure_session,zeek.hart_ip_session_record.session_log_record_session_status_summary_session_timeout,zeek.hart_ip_session_record.session_log_record_session_status_summary_aborted_session,zeek.hart_ip_session_record.session_log_record_session_status_summary_bad_session_initialization,zeek.hart_ip_session_record.session_log_record_session_status_summary_writes_occured,zeek.hart_ip_session_record.session_log_record_start_configuration_change_count,zeek.hart_ip_session_record.session_log_record_end_configuration_change_count,zeek.hart_ip_session_record.session_log_record_num_publish_pdu,zeek.hart_ip_session_record.session_log_record_num_request_pdu,zeek.hart_ip_session_record.session_log_record_num_response_pdu o_zeek_hart_ip_universal_commands=require:zeek.hart_ip_universal_commands;title:Zeek hart_ip_universal_commands.log;fields:zeek.hart_ip.command_number_link_id,zeek.hart_ip_universal_commands.read_unique_identifier_response_254,zeek.hart_ip_universal_commands.read_unique_identifier_response_expanded_device_type,zeek.hart_ip_universal_commands.read_unique_identifier_response_minimum_preambles_master_slave,zeek.hart_ip_universal_commands.read_unique_identifier_response_hart_protocol_major_revision,zeek.hart_ip_universal_commands.read_unique_identifier_response_device_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_response_software_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_response_hardware_revision_level_and_physical_signaling_codes_hardware_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_response_hardware_revision_level_and_physical_signaling_codes_physical_signaling_code,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_c8_psk_in_multi_drop_only,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_c8_psk_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_undefined_5,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_safehart_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_ieee_802_15_4_dsss_o_qpsk_modulation,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_protocol_bridge_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_eeprom_control,zeek.hart_ip_universal_commands.read_unique_identifier_response_flags_mutli_sensor_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_response_device_id,zeek.hart_ip_universal_commands.read_unique_identifier_response_number_preambles_slave_master,zeek.hart_ip_universal_commands.read_unique_identifier_response_last_device_variable_this,zeek.hart_ip_universal_commands.read_unique_identifier_response_configuration_change_counter,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_unique_identifier_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_unique_identifier_response_manufacturer_identification_code,zeek.hart_ip_universal_commands.read_unique_identifier_response_private_label_distributor_code,zeek.hart_ip_universal_commands.read_unique_identifier_response_device_profile,zeek.hart_ip_universal_commands.read_primary_variable_response_primary_variable_units,zeek.hart_ip_universal_commands.read_primary_variable_response_primary_variable,zeek.hart_ip_universal_commands.read_loop_current_response_primary_variable_loop_current,zeek.hart_ip_universal_commands.read_loop_current_response_primary_variable_percent_range,zeek.hart_ip_universal_commands.read_dynamic_variable_response_primary_variable_loop_current,zeek.hart_ip_universal_commands.read_dynamic_variable_response_primary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_primary_variable,zeek.hart_ip_universal_commands.read_dynamic_variable_response_secondary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_secondary_variable,zeek.hart_ip_universal_commands.read_dynamic_variable_response_tertiary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_tertiary_variable,zeek.hart_ip_universal_commands.read_dynamic_variable_response_quaternary_variable_units,zeek.hart_ip_universal_commands.read_dynamic_variable_response_quaternary_variable,zeek.hart_ip_universal_commands.write_polling_address_polling_address_device,zeek.hart_ip_universal_commands.write_polling_address_loop_current_mode,zeek.hart_ip_universal_commands.read_loop_configuration_response_polling_address_device,zeek.hart_ip_universal_commands.read_loop_configuration_response_loop_current_mode,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_primary_variable_classification,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_secondary_variable_classification,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_tertiary_variable_classification,zeek.hart_ip_universal_commands.read_dynamic_variable_classifications_response_quaternary_variable_classification,zeek.hart_ip_universal_commands.read_device_variable_request_slot0_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot1_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot2_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot3_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot4_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot5_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot6_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_request_slot7_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_device_variable_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot1_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot2_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot3_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot4_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot5_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot6_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_class,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_units_code,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_process_data_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_limit_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_more_device_variable_status_available,zeek.hart_ip_universal_commands.read_device_variable_response_slot7_device_variable_status_device_family_specific_status,zeek.hart_ip_universal_commands.read_device_variable_response_slot0_time,zeek.hart_ip_universal_commands.read_unique_identifier_tag_request_tag,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_254,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_expanded_device_type,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_minimum_preambles_master_slave,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_hart_protocol_major_revision,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_device_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_software_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_hardware_revision_level_and_physical_signaling_codes_hardware_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_hardware_revision_level_and_physical_signaling_codes_physical_signaling_code,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_c8_psk_in_multi_drop_only,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_c8_psk_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_undefined_5,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_safehart_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_ieee_802_15_4_dsss_o_qpsk_modulation,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_protocol_bridge_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_eeprom_control,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_flags_mutli_sensor_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_device_id,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_number_preambles_slave_master,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_last_device_variable_this,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_configuration_change_counter,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_manufacturer_identification_code,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_private_label_distributor_code,zeek.hart_ip_universal_commands.read_unique_identifier_tag_response_device_profile,zeek.hart_ip_universal_commands.read_message_response_message,zeek.hart_ip_universal_commands.read_tag_response_tag,zeek.hart_ip_universal_commands.read_tag_response_descriptor,zeek.hart_ip_universal_commands.read_tag_response_date_code,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_transducer_serial_number,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_transducer_limits_units,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_upper_transducer_limit,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_lower_transducer_limit,zeek.hart_ip_universal_commands.read_primary_variable_transducer_information_response_p_v_minimum_span,zeek.hart_ip_universal_commands.read_device_information_response_p_v_alarm_selection_code,zeek.hart_ip_universal_commands.read_device_information_response_p_v_transfer_function_code,zeek.hart_ip_universal_commands.read_device_information_response_p_v_upper_lower_range,zeek.hart_ip_universal_commands.read_device_information_response_p_v_upper_range_value,zeek.hart_ip_universal_commands.read_device_information_response_p_v_lower_range_value,zeek.hart_ip_universal_commands.read_device_information_response_p_v_damping_value,zeek.hart_ip_universal_commands.read_device_information_response_write_protect_code,zeek.hart_ip_universal_commands.read_device_information_response_250,zeek.hart_ip_universal_commands.read_device_information_response_p_v_analog_channel_flags_undefined_bits,zeek.hart_ip_universal_commands.read_device_information_response_p_v_analog_channel_flags_analog_channel,zeek.hart_ip_universal_commands.read_final_assembly_number_response_final_assembly_number,zeek.hart_ip_universal_commands.write_message_message_string,zeek.hart_ip_universal_commands.write_tag_descriptor_date_tag,zeek.hart_ip_universal_commands.write_tag_descriptor_date_record_keeping_descriptor,zeek.hart_ip_universal_commands.write_tag_descriptor_date_date_code,zeek.hart_ip_universal_commands.write_final_assembly_number_final_assembly_number,zeek.hart_ip_universal_commands.read_long_tag_response_long_tag,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_request_long_tag,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_254,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_expanded_device_type,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_minimum_preambles_master_slave,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_hart_protocol_major_revision,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_device_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_software_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_hardware_revision_level_and_physical_signaling_codes_hardware_revision_level,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_hardware_revision_level_and_physical_signaling_codes_physical_signaling_code,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_c8_psk_in_multi_drop_only,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_c8_psk_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_undefined_5,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_safehart_capable_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_ieee_802_15_4_dsss_o_qpsk_modulation,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_protocol_bridge_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_eeprom_control,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_flags_mutli_sensor_field_device,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_device_id,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_number_preambles_slave_master,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_last_device_variable_this,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_configuration_change_counter,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_manufacturer_identification_code,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_private_label_distributor_code,zeek.hart_ip_universal_commands.read_unique_identifier_long_tag_response_device_profile,zeek.hart_ip_universal_commands.write_long_tag_long_tag,zeek.hart_ip_universal_commands.reset_configuration_changed_flag_configuration_change_counter,zeek.hart_ip_universal_commands.read_additional_device_status_contents_device_specific_status_0,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_function_check,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_out_of_specification,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_failure,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_critical_power_failure,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_device_variable_alert,zeek.hart_ip_universal_commands.read_additional_device_status_contents_extended_field_device_status_maintenance_required,zeek.hart_ip_universal_commands.read_additional_device_status_contents_device_operating_mode,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_device_configuration_lock,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_electronic_defect,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_environmental_conditions_out_of_range,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_power_supply_conditions_out_of_range,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_watchdog_reset_executed,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_volatile_memory_defect,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_non_volatile_memory_defect,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status0_device_variable_simulation_active,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_reserved,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_battery_or_power_supply_needs_maintenance,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_event_notification_overflow,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_discrete_variable_simulation_active,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status1_status_simulation_active,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_quinary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_quaternary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_tertiary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_saturated_secondary_analog,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_stale_data_notice,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_sub_device_with_duplicate_id,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_sub_device_mismatch,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_duplicate_master_detected,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status2_sub_device_list_changed,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_radio_failure,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_block_transfer_pending,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_bandwith_allocation_pending,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_resereved,zeek.hart_ip_universal_commands.read_additional_device_status_contents_standardized_status3_capacity_denied,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_undefined_bits,zeek.hart_ip_universal_commands.read_additional_device_status_contents_analog_channel_analog_channel,zeek.hart_ip_universal_commands.read_additional_device_status_contents_device_specific_status_1 o_zeek_http=require:zeek.http;title:Zeek http.log;fields:zeek.http.trans_depth,zeek.http.method,zeek.http.host,zeek.http.uri,zeek.http.origin,zeek.http.post_password_plain,zeek.http.post_username,zeek.http.referrer,zeek.http.version,zeek.http.user_agent,zeek.http.request_body_len,zeek.http.response_body_len,zeek.http.status_code,zeek.http.status_msg,zeek.http.info_code,zeek.http.info_msg,zeek.http.tags,zeek.http.proxied,zeek.http.orig_fuids,zeek.http.orig_filenames,zeek.http.orig_mime_types,zeek.http.resp_fuids,zeek.http.resp_filenames,zeek.http.resp_mime_types,zeek.http.ja4h,zeek.http.client_header_names,zeek.http.server_header_names -o_zeek_intel=require:zeek.intel;title:Zeek intel.log;fields:zeek.intel.cif_tags,zeek.intel.cif_confidence,zeek.intel.cif_source,zeek.intel.cif_description,zeek.intel.cif_firstseen,zeek.intel.cif_lastseen,zeek.intel.seen_indicator,zeek.intel.seen_indicator_type,zeek.intel.seen_where,zeek.intel.seen_node,zeek.intel.matched,zeek.intel.sources,zeek.intel.file_mime_type,zeek.intel.file_description +o_zeek_intel=require:zeek.intel;title:Zeek intel.log;fields:tags,zeek.intel.cif_confidence,threat.indicator.provider,threat.indicator.description,threat.indicator.first_seen,threat.indicator.last_seen,threat.indicator.name,threat.indicator.type,zeek.intel.seen_where,zeek.intel.seen_node,zeek.intel.file_mime_type,zeek.intel.file_description o_zeek_ipsec=require:zeek.ipsec;title:Zeek ipsec.log;fields:zeek.ipsec.initiator_spi,zeek.ipsec.responder_spi,zeek.ipsec.maj_ver,zeek.ipsec.min_ver,zeek.ipsec.exchange_type,zeek.ipsec.doi,zeek.ipsec.flag_e,zeek.ipsec.flag_c,zeek.ipsec.flag_a,zeek.ipsec.flag_i,zeek.ipsec.flag_v,zeek.ipsec.flag_r,zeek.ipsec.flags,zeek.ipsec.message_id,zeek.ipsec.vendor_ids,zeek.ipsec.notify_messages,zeek.ipsec.transforms,zeek.ipsec.ke_dh_groups,zeek.ipsec.proposals,zeek.ipsec.protocol_id,zeek.ipsec.certificates,zeek.ipsec.situation,zeek.ipsec.transform_attributes,zeek.ipsec.length,zeek.ipsec.hash o_zeek_irc=require:zeek.irc;title:Zeek irc.log;fields:zeek.irc.nick,zeek.irc.command,zeek.irc.value,zeek.irc.addl,zeek.irc.dcc_file_name,zeek.irc.dcc_file_size,zeek.irc.dcc_mime_type o_zeek_kerberos=require:zeek.kerberos;title:Zeek kerberos.log;fields:zeek.kerberos.cname,zeek.kerberos.sname,zeek.kerberos.success,zeek.kerberos.error_msg,zeek.kerberos.from,zeek.kerberos.till,zeek.kerberos.cipher,zeek.kerberos.forwardable,zeek.kerberos.renewable,zeek.kerberos.request_type,zeek.kerberos.client_cert_subject,zeek.kerberos.client_cert_fuid,zeek.kerberos.server_cert_subject,zeek.kerberos.server_cert_fuid @@ -3366,7 +3375,7 @@ o_zeek_notice=require:zeek.notice;title:Zeek notice.log;fields:rule.category,rul o_zeek_ntlm=require:zeek.ntlm;title:Zeek ntlm.log;fields:zeek.ntlm.host,zeek.ntlm.domain,zeek.ntlm.success,zeek.ntlm.status,zeek.ntlm.server_nb_computer,zeek.ntlm.server_dns_computer,zeek.ntlm.server_tree o_zeek_ntp=require:zeek.ntp;title:Zeek ntp.log;fields:zeek.ntp.version,zeek.ntp.mode,zeek.ntp.mode_str,zeek.ntp.stratum,zeek.ntp.poll,zeek.ntp.precision,zeek.ntp.root_delay,zeek.ntp.root_disp,zeek.ntp.ref_id,zeek.ntp.ref_time,zeek.ntp.org_time,zeek.ntp.rec_time,zeek.ntp.xmt_time,zeek.ntp.num_exts o_zeek_ocsp=require:zeek.ocsp;title:Zeek ocsp.log;fields:zeek.ocsp.hashAlgorithm,zeek.ocsp.issuerNameHash,zeek.ocsp.issuerKeyHash,zeek.ocsp.serialNumber,zeek.ocsp.certStatus,zeek.ocsp.revoketime,zeek.ocsp.revokereason,zeek.ocsp.thisUpdate,zeek.ocsp.nextUpdate -o_zeek_opcua=require:zeek.opcua_binary;title:Zeek OPC UA Binary logs;fields:zeek.opcua_binary.filter_source_link_id,zeek.opcua_binary.operand_source_link_id,zeek.opcua_binary.variant_source_link_id,zeek.opcua_binary.encoding_mask,zeek.opcua_binary.endpoint_url,zeek.opcua_binary.error,zeek.opcua_binary.identifier,zeek.opcua_binary.identifier_str,zeek.opcua_binary.is_final,zeek.opcua_binary.max_chunk_cnt,zeek.opcua_binary.max_msg_size,zeek.opcua_binary.msg_size,zeek.opcua_binary.msg_type,zeek.opcua_binary.namespace_idx,zeek.opcua_binary.opcua_link_id,zeek.opcua_binary.rcv_buf_size,zeek.opcua_binary.rcv_cert,zeek.opcua_binary.rcv_cert_len,zeek.opcua_binary.reason,zeek.opcua_binary.req_hdr_add_hdr_enc_mask,zeek.opcua_binary.req_hdr_add_hdr_type_id,zeek.opcua_binary.req_hdr_audit_entry_id,zeek.opcua_binary.req_hdr_node_id_guid,zeek.opcua_binary.req_hdr_node_id_namespace_idx,zeek.opcua_binary.req_hdr_node_id_numeric,zeek.opcua_binary.req_hdr_node_id_opaque,zeek.opcua_binary.req_hdr_node_id_string,zeek.opcua_binary.req_hdr_node_id_type,zeek.opcua_binary.req_hdr_request_handle,zeek.opcua_binary.req_hdr_return_diag,zeek.opcua_binary.req_hdr_timeout_hint,zeek.opcua_binary.req_hdr_timestamp,zeek.opcua_binary.request_id,zeek.opcua_binary.res_hdr_add_hdr_enc_mask,zeek.opcua_binary.res_hdr_add_hdr_type_id,zeek.opcua_binary.res_hdr_request_handle,zeek.opcua_binary.res_hdr_service_diag_encoding,zeek.opcua_binary.res_hdr_timestamp,zeek.opcua_binary.sec_channel_id,zeek.opcua_binary.sec_policy_uri,zeek.opcua_binary.sec_policy_uri_len,zeek.opcua_binary.seq_number,zeek.opcua_binary.snd_buf_size,zeek.opcua_binary.snd_cert,zeek.opcua_binary.snd_cert_len,zeek.opcua_binary.version,zeek.opcua_binary_activate_session.client_algorithm,zeek.opcua_binary_activate_session.client_signature,zeek.opcua_binary_activate_session.ext_obj_certificate_data,zeek.opcua_binary_activate_session.ext_obj_encoding,zeek.opcua_binary_activate_session.ext_obj_encryption_algorithom,zeek.opcua_binary_activate_session.ext_obj_password,zeek.opcua_binary_activate_session.ext_obj_policy_id,zeek.opcua_binary_activate_session.ext_obj_token_data,zeek.opcua_binary_activate_session.ext_obj_type_id_encoding_mask,zeek.opcua_binary_activate_session.ext_obj_type_id_guid,zeek.opcua_binary_activate_session.ext_obj_type_id_namespace_idx,zeek.opcua_binary_activate_session.ext_obj_type_id_numeric,zeek.opcua_binary_activate_session.ext_obj_type_id_opaque,zeek.opcua_binary_activate_session.ext_obj_type_id_str,zeek.opcua_binary_activate_session.ext_obj_type_id_string,zeek.opcua_binary_activate_session.ext_obj_user_name,zeek.opcua_binary_activate_session.server_nonce,zeek.opcua_binary_activate_session.user_token_algorithm,zeek.opcua_binary_activate_session.user_token_signature,zeek.opcua_binary_activate_session_client_software_cert.cert_data,zeek.opcua_binary_activate_session_client_software_cert.cert_signature,zeek.opcua_binary_activate_session_client_software_cert.client_software_cert_link_id,zeek.opcua_binary_activate_session_locale_id.local_id,zeek.opcua_binary_activate_session_locale_id.opcua_locale_link_id,zeek.opcua_binary_aggregate_filter.aggregate_type_encoding_mask,zeek.opcua_binary_aggregate_filter.aggregate_type_guid,zeek.opcua_binary_aggregate_filter.aggregate_type_namespace_idx,zeek.opcua_binary_aggregate_filter.aggregate_type_numeric,zeek.opcua_binary_aggregate_filter.aggregate_type_opaque,zeek.opcua_binary_aggregate_filter.aggregate_type_string,zeek.opcua_binary_aggregate_filter.percent_data_bad,zeek.opcua_binary_aggregate_filter.percent_data_good,zeek.opcua_binary_aggregate_filter.processing_interval,zeek.opcua_binary_aggregate_filter.revised_percent_data_bad,zeek.opcua_binary_aggregate_filter.revised_percent_data_good,zeek.opcua_binary_aggregate_filter.revised_processing_interval,zeek.opcua_binary_aggregate_filter.revised_start_time,zeek.opcua_binary_aggregate_filter.revised_start_time_str,zeek.opcua_binary_aggregate_filter.revised_treat_uncertain_as_bad,zeek.opcua_binary_aggregate_filter.revised_use_server_capabilities_default,zeek.opcua_binary_aggregate_filter.revised_use_slopped_extrapolation,zeek.opcua_binary_aggregate_filter.start_time,zeek.opcua_binary_aggregate_filter.start_time_str,zeek.opcua_binary_aggregate_filter.treat_uncertain_as_bad,zeek.opcua_binary_aggregate_filter.use_server_capabilities_default,zeek.opcua_binary_aggregate_filter.use_slopped_extrapolation,zeek.opcua_binary_browse.browse_next_release_continuation_point,zeek.opcua_binary_browse.browse_service_type,zeek.opcua_binary_browse.browse_view_description_timestamp,zeek.opcua_binary_browse.browse_view_description_view_version,zeek.opcua_binary_browse.browse_view_id_encoding_mask,zeek.opcua_binary_browse.browse_view_id_guid,zeek.opcua_binary_browse.browse_view_id_namespace_idx,zeek.opcua_binary_browse.browse_view_id_numeric,zeek.opcua_binary_browse.browse_view_id_opaque,zeek.opcua_binary_browse.browse_view_id_string,zeek.opcua_binary_browse.req_max_ref_nodes,zeek.opcua_binary_browse_description.browse_description_encoding_mask,zeek.opcua_binary_browse_description.browse_description_guid,zeek.opcua_binary_browse_description.browse_description_include_subtypes,zeek.opcua_binary_browse_description.browse_description_link_id,zeek.opcua_binary_browse_description.browse_description_namespace_idx,zeek.opcua_binary_browse_description.browse_description_numeric,zeek.opcua_binary_browse_description.browse_description_opaque,zeek.opcua_binary_browse_description.browse_description_ref_encoding_mask,zeek.opcua_binary_browse_description.browse_description_ref_guid,zeek.opcua_binary_browse_description.browse_description_ref_namespace_idx,zeek.opcua_binary_browse_description.browse_description_ref_numeric,zeek.opcua_binary_browse_description.browse_description_ref_opaque,zeek.opcua_binary_browse_description.browse_description_ref_string,zeek.opcua_binary_browse_description.browse_description_string,zeek.opcua_binary_browse_description.browse_direction,zeek.opcua_binary_browse_description.browse_node_class_mask,zeek.opcua_binary_browse_description.browse_result_mask,zeek.opcua_binary_browse_request_continuation_point.browse_next_link_id,zeek.opcua_binary_browse_request_continuation_point.continuation_point,zeek.opcua_binary_browse_response_references.browse_reference_link_id,zeek.opcua_binary_browse_response_references.browse_response_display_name_locale,zeek.opcua_binary_browse_response_references.browse_response_display_name_mask,zeek.opcua_binary_browse_response_references.browse_response_display_name_text,zeek.opcua_binary_browse_response_references.browse_response_is_forward,zeek.opcua_binary_browse_response_references.browse_response_node_class,zeek.opcua_binary_browse_response_references.browse_response_ref_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_ref_guid,zeek.opcua_binary_browse_response_references.browse_response_ref_name,zeek.opcua_binary_browse_response_references.browse_response_ref_name_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_numeric,zeek.opcua_binary_browse_response_references.browse_response_ref_opaque,zeek.opcua_binary_browse_response_references.browse_response_ref_string,zeek.opcua_binary_browse_response_references.browse_response_ref_type_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_ref_type_guid,zeek.opcua_binary_browse_response_references.browse_response_ref_type_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_type_namespace_uri,zeek.opcua_binary_browse_response_references.browse_response_ref_type_numeric,zeek.opcua_binary_browse_response_references.browse_response_ref_type_opaque,zeek.opcua_binary_browse_response_references.browse_response_ref_type_server_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_type_string,zeek.opcua_binary_browse_response_references.browse_response_type_def_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_type_def_guid,zeek.opcua_binary_browse_response_references.browse_response_type_def_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_type_def_namespace_uri,zeek.opcua_binary_browse_response_references.browse_response_type_def_numeric,zeek.opcua_binary_browse_response_references.browse_response_type_def_opaque,zeek.opcua_binary_browse_response_references.browse_response_type_def_server_idx,zeek.opcua_binary_browse_response_references.browse_response_type_def_string,zeek.opcua_binary_browse_result.browse_response_link_id,zeek.opcua_binary_browse_result.browse_result_continuation_point,zeek.opcua_binary_close_session.del_subscriptions,zeek.opcua_binary_create_monitored_items.subscription_id,zeek.opcua_binary_create_monitored_items.timestamps_to_return,zeek.opcua_binary_create_monitored_items.timestamps_to_return_str,zeek.opcua_binary_create_monitored_items_create_item.create_item_link_id,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_attribute_id,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_index_range,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_name,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_encoding_mask,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_guid,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_numeric,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_opaque,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitored_item_index_id,zeek.opcua_binary_create_monitored_items_create_item.monitoring_mode,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_client_handle,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_discard_oldest,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_encoding,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_encoding_mask,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_guid,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_numeric,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_opaque,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_queue_size,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_revised_queue_size,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_revised_sampling_interval,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_sampling_interval,zeek.opcua_binary_create_session.algorithm,zeek.opcua_binary_create_session.application_type,zeek.opcua_binary_create_session.application_uri,zeek.opcua_binary_create_session.auth_token_encoding_mask,zeek.opcua_binary_create_session.auth_token_guid,zeek.opcua_binary_create_session.auth_token_namespace_idx,zeek.opcua_binary_create_session.auth_token_numeric,zeek.opcua_binary_create_session.auth_token_opaque,zeek.opcua_binary_create_session.auth_token_string,zeek.opcua_binary_create_session.client_cert,zeek.opcua_binary_create_session.client_cert_size,zeek.opcua_binary_create_session.client_nonce,zeek.opcua_binary_create_session.discovery_profile_uri,zeek.opcua_binary_create_session.encoding_mask,zeek.opcua_binary_create_session.endpoint_url,zeek.opcua_binary_create_session.gateway_server_uri,zeek.opcua_binary_create_session.locale,zeek.opcua_binary_create_session.max_req_msg_size,zeek.opcua_binary_create_session.max_res_msg_size,zeek.opcua_binary_create_session.product_uri,zeek.opcua_binary_create_session.req_session_timeout,zeek.opcua_binary_create_session.revised_session_timeout,zeek.opcua_binary_create_session.server_cert,zeek.opcua_binary_create_session.server_cert_size,zeek.opcua_binary_create_session.server_nonce,zeek.opcua_binary_create_session.server_uri,zeek.opcua_binary_create_session.session_id_encoding_mask,zeek.opcua_binary_create_session.session_id_guid,zeek.opcua_binary_create_session.session_id_namespace_idx,zeek.opcua_binary_create_session.session_id_numeric,zeek.opcua_binary_create_session.session_id_opaque,zeek.opcua_binary_create_session.session_id_string,zeek.opcua_binary_create_session.session_name,zeek.opcua_binary_create_session.signature,zeek.opcua_binary_create_session.text,zeek.opcua_binary_create_session_discovery.discovery_profile_link_id,zeek.opcua_binary_create_session_discovery.discovery_profile_uri,zeek.opcua_binary_create_session_discovery.discovery_profile_url,zeek.opcua_binary_create_session_endpoints.application_type,zeek.opcua_binary_create_session_endpoints.application_uri,zeek.opcua_binary_create_session_endpoints.cert_size,zeek.opcua_binary_create_session_endpoints.discovery_profile_uri,zeek.opcua_binary_create_session_endpoints.encoding_mask,zeek.opcua_binary_create_session_endpoints.endpoint_link_id,zeek.opcua_binary_create_session_endpoints.endpoint_url,zeek.opcua_binary_create_session_endpoints.gateway_server_uri,zeek.opcua_binary_create_session_endpoints.locale,zeek.opcua_binary_create_session_endpoints.message_security_mode,zeek.opcua_binary_create_session_endpoints.product_uri,zeek.opcua_binary_create_session_endpoints.security_level,zeek.opcua_binary_create_session_endpoints.security_policy_uri,zeek.opcua_binary_create_session_endpoints.server_cert,zeek.opcua_binary_create_session_endpoints.text,zeek.opcua_binary_create_session_endpoints.transport_profile_uri,zeek.opcua_binary_create_session_user_token.user_token_endpoint_url,zeek.opcua_binary_create_session_user_token.user_token_issued_type,zeek.opcua_binary_create_session_user_token.user_token_link_id,zeek.opcua_binary_create_session_user_token.user_token_policy_id,zeek.opcua_binary_create_session_user_token.user_token_sec_policy_uri,zeek.opcua_binary_create_session_user_token.user_token_type,zeek.opcua_binary_create_subscription.max_notifications_per_publish,zeek.opcua_binary_create_subscription.priority,zeek.opcua_binary_create_subscription.publishing_enabled,zeek.opcua_binary_create_subscription.requested_lifetime_count,zeek.opcua_binary_create_subscription.requested_max_keep_alive_count,zeek.opcua_binary_create_subscription.requested_publishing_interval,zeek.opcua_binary_create_subscription.revised_lifetime_count,zeek.opcua_binary_create_subscription.revised_max_keep_alive_count,zeek.opcua_binary_create_subscription.revised_publishing_interval,zeek.opcua_binary_create_subscription.subscription_id,zeek.opcua_binary_data_change_filter.deadband_type,zeek.opcua_binary_data_change_filter.deadband_value,zeek.opcua_binary_data_change_filter.trigger,zeek.opcua_binary_diag_info_detail.addl_info,zeek.opcua_binary_diag_info_detail.diag_info_link_id,zeek.opcua_binary_diag_info_detail.has_addl_info,zeek.opcua_binary_diag_info_detail.has_inner_diag_info,zeek.opcua_binary_diag_info_detail.has_inner_stat_code,zeek.opcua_binary_diag_info_detail.has_locale,zeek.opcua_binary_diag_info_detail.has_locale_txt,zeek.opcua_binary_diag_info_detail.has_namespace_uri,zeek.opcua_binary_diag_info_detail.has_symbolic_id,zeek.opcua_binary_diag_info_detail.inner_diag_level,zeek.opcua_binary_diag_info_detail.inner_stat_code,zeek.opcua_binary_diag_info_detail.locale,zeek.opcua_binary_diag_info_detail.locale_str,zeek.opcua_binary_diag_info_detail.locale_txt,zeek.opcua_binary_diag_info_detail.locale_txt_str,zeek.opcua_binary_diag_info_detail.namespace_uri,zeek.opcua_binary_diag_info_detail.namespace_uri_str,zeek.opcua_binary_diag_info_detail.root_object_id,zeek.opcua_binary_diag_info_detail.source,zeek.opcua_binary_diag_info_detail.source_str,zeek.opcua_binary_diag_info_detail.symbolic_id,zeek.opcua_binary_diag_info_detail.symbolic_id_str,zeek.opcua_binary_event_filter_attribute_operand.alias,zeek.opcua_binary_event_filter_attribute_operand.attribute,zeek.opcua_binary_event_filter_attribute_operand.index_range,zeek.opcua_binary_event_filter_attribute_operand.node_id_encoding_mask,zeek.opcua_binary_event_filter_attribute_operand.node_id_guid,zeek.opcua_binary_event_filter_attribute_operand.node_id_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand.node_id_numeric,zeek.opcua_binary_event_filter_attribute_operand.node_id_opaque,zeek.opcua_binary_event_filter_attribute_operand.node_id_string,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.browse_path_element_link_id,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.include_subtypes,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.is_inverse,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.target_name,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.target_name_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_encoding_mask,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_guid,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_numeric,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_opaque,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_string,zeek.opcua_binary_event_filter_element_operand.element_index,zeek.opcua_binary_event_filter_select_clause.attribute_id,zeek.opcua_binary_event_filter_select_clause.index_range,zeek.opcua_binary_event_filter_select_clause.select_clause_link_id,zeek.opcua_binary_event_filter_select_clause.type_id_encoding_mask,zeek.opcua_binary_event_filter_select_clause.type_id_guid,zeek.opcua_binary_event_filter_select_clause.type_id_namespace_idx,zeek.opcua_binary_event_filter_select_clause.type_id_numeric,zeek.opcua_binary_event_filter_select_clause.type_id_opaque,zeek.opcua_binary_event_filter_select_clause.type_id_string,zeek.opcua_binary_event_filter_simple_attribute_operand.attribute_id,zeek.opcua_binary_event_filter_simple_attribute_operand.index_range,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_encoding_mask,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_guid,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_namespace_idx,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_numeric,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_opaque,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_string,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.browse_path_src,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.name,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.namespace_index,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.simple_attribute_operand_browse_path_link_id,zeek.opcua_binary_event_filter_where_clause.where_clause_link_id,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_element_link_id,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_encoding,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_encoding_mask,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_guid,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_namespace_idx,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_numeric,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_opaque,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_string,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_string,zeek.opcua_binary_event_filter_where_clause_elements.filter_operator,zeek.opcua_binary_get_endpoints.endpoint_url,zeek.opcua_binary_get_endpoints_description.application_type,zeek.opcua_binary_get_endpoints_description.application_uri,zeek.opcua_binary_get_endpoints_description.cert_size,zeek.opcua_binary_get_endpoints_description.discovery_profile_uri,zeek.opcua_binary_get_endpoints_description.encoding_mask,zeek.opcua_binary_get_endpoints_description.endpoint_description_link_id,zeek.opcua_binary_get_endpoints_description.endpoint_uri,zeek.opcua_binary_get_endpoints_description.gateway_server_uri,zeek.opcua_binary_get_endpoints_description.locale,zeek.opcua_binary_get_endpoints_description.message_security_mode,zeek.opcua_binary_get_endpoints_description.product_uri,zeek.opcua_binary_get_endpoints_description.security_level,zeek.opcua_binary_get_endpoints_description.security_policy_uri,zeek.opcua_binary_get_endpoints_description.server_cert,zeek.opcua_binary_get_endpoints_description.text,zeek.opcua_binary_get_endpoints_description.transport_profile_uri,zeek.opcua_binary_get_endpoints_discovery.discovery_profile_link_id,zeek.opcua_binary_get_endpoints_discovery.discovery_profile_url,zeek.opcua_binary_get_endpoints_locale_id.locale_id,zeek.opcua_binary_get_endpoints_locale_id.locale_link_id,zeek.opcua_binary_get_endpoints_profile_uri.profile_uri,zeek.opcua_binary_get_endpoints_profile_uri.profile_uri_link_id,zeek.opcua_binary_get_endpoints_user_token.user_token_endpoint_url,zeek.opcua_binary_get_endpoints_user_token.user_token_issued_type,zeek.opcua_binary_get_endpoints_user_token.user_token_link_id,zeek.opcua_binary_get_endpoints_user_token.user_token_policy_id,zeek.opcua_binary_get_endpoints_user_token.user_token_sec_policy_uri,zeek.opcua_binary_get_endpoints_user_token.user_token_type,zeek.opcua_binary_opensecure_channel.client_nonce,zeek.opcua_binary_opensecure_channel.client_proto_ver,zeek.opcua_binary_opensecure_channel.message_security_mode,zeek.opcua_binary_opensecure_channel.req_lifetime,zeek.opcua_binary_opensecure_channel.sec_token_created_at,zeek.opcua_binary_opensecure_channel.sec_token_id,zeek.opcua_binary_opensecure_channel.sec_token_request_type,zeek.opcua_binary_opensecure_channel.sec_token_revised_time,zeek.opcua_binary_opensecure_channel.sec_token_sec_channel_id,zeek.opcua_binary_opensecure_channel.server_nonce,zeek.opcua_binary_opensecure_channel.server_proto_ver,zeek.opcua_binary_read.max_age,zeek.opcua_binary_read.timestamps_to_return,zeek.opcua_binary_read.timestamps_to_return_str,zeek.opcua_binary_read_nodes_to_read.attribute_id,zeek.opcua_binary_read_nodes_to_read.attribute_id_str,zeek.opcua_binary_read_nodes_to_read.data_encoding_name,zeek.opcua_binary_read_nodes_to_read.data_encoding_name_idx,zeek.opcua_binary_read_nodes_to_read.index_range,zeek.opcua_binary_read_nodes_to_read.node_id_encoding_mask,zeek.opcua_binary_read_nodes_to_read.node_id_guid,zeek.opcua_binary_read_nodes_to_read.node_id_namespace_idx,zeek.opcua_binary_read_nodes_to_read.node_id_numeric,zeek.opcua_binary_read_nodes_to_read.node_id_opaque,zeek.opcua_binary_read_nodes_to_read.node_id_string,zeek.opcua_binary_read_nodes_to_read.nodes_to_read_link_id,zeek.opcua_binary_read_results.data_value_encoding_mask,zeek.opcua_binary_read_results.level,zeek.opcua_binary_read_results.results_link_id,zeek.opcua_binary_read_results.server_pico_sec,zeek.opcua_binary_read_results.server_timestamp,zeek.opcua_binary_read_results.source_pico_sec,zeek.opcua_binary_read_results.source_timestamp,zeek.opcua_binary_status_code_detail.historian_bits,zeek.opcua_binary_status_code_detail.historian_bits_str,zeek.opcua_binary_status_code_detail.historianextradata,zeek.opcua_binary_status_code_detail.historianmultivalue,zeek.opcua_binary_status_code_detail.historianpartial,zeek.opcua_binary_status_code_detail.info_type,zeek.opcua_binary_status_code_detail.info_type_str,zeek.opcua_binary_status_code_detail.limit_bits,zeek.opcua_binary_status_code_detail.limit_bits_str,zeek.opcua_binary_status_code_detail.overflow,zeek.opcua_binary_status_code_detail.semantics_changed,zeek.opcua_binary_status_code_detail.severity,zeek.opcua_binary_status_code_detail.severity_str,zeek.opcua_binary_status_code_detail.source,zeek.opcua_binary_status_code_detail.source_level,zeek.opcua_binary_status_code_detail.source_str,zeek.opcua_binary_status_code_detail.status_code,zeek.opcua_binary_status_code_detail.status_code_link_id,zeek.opcua_binary_status_code_detail.structure_changed,zeek.opcua_binary_status_code_detail.sub_code,zeek.opcua_binary_status_code_detail.sub_code_str,zeek.opcua_binary_variant_array_dims.array_dim_link_id,zeek.opcua_binary_variant_array_dims.dimension,zeek.opcua_binary_variant_data.variant_data_encoding_name,zeek.opcua_binary_variant_data.variant_data_encoding_name_idx,zeek.opcua_binary_variant_data.variant_data_link_id,zeek.opcua_binary_variant_data.variant_data_locale,zeek.opcua_binary_variant_data.variant_data_mask,zeek.opcua_binary_variant_data.variant_data_node_id_encoding_mask,zeek.opcua_binary_variant_data.variant_data_node_id_guid,zeek.opcua_binary_variant_data.variant_data_node_id_namespace_idx,zeek.opcua_binary_variant_data.variant_data_node_id_namespace_uri,zeek.opcua_binary_variant_data.variant_data_node_id_numeric,zeek.opcua_binary_variant_data.variant_data_node_id_opaque,zeek.opcua_binary_variant_data.variant_data_node_id_server_idx,zeek.opcua_binary_variant_data.variant_data_node_id_string,zeek.opcua_binary_variant_data.variant_data_text,zeek.opcua_binary_variant_data.variant_data_value_decimal,zeek.opcua_binary_variant_data.variant_data_value_signed_numeric,zeek.opcua_binary_variant_data.variant_data_value_string,zeek.opcua_binary_variant_data.variant_data_value_time,zeek.opcua_binary_variant_data.variant_data_value_unsigned_numeric,zeek.opcua_binary_variant_data_value.data_value_encoding_mask,zeek.opcua_binary_variant_data_value.server_pico_sec,zeek.opcua_binary_variant_data_value.server_timestamp,zeek.opcua_binary_variant_data_value.source_pico_sec,zeek.opcua_binary_variant_data_value.source_timestamp,zeek.opcua_binary_variant_data_value.variant_data_value_source_link,zeek.opcua_binary_variant_extension_object.ext_obj_encoding,zeek.opcua_binary_variant_extension_object.ext_obj_link_id,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_encoding_mask,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_guid,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_namespace_idx,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_numeric,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_opaque,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_string,zeek.opcua_binary_variant_extension_object.ext_obj_type_id_str,zeek.opcua_binary_variant_metadata.built_in_data_type,zeek.opcua_binary_variant_metadata.built_in_data_type_str,zeek.opcua_binary_variant_metadata.dara_variant_encoding_mask,zeek.opcua_binary_variant_metadata.data_variant_data_type,zeek.opcua_binary_variant_metadata.data_variant_data_type_str,zeek.opcua_binary_variant_metadata.variant_data_array_dim,zeek.opcua_binary_variant_metadata.variant_data_source,zeek.opcua_binary_variant_metadata.variant_data_source_str +o_zeek_opcua=require:zeek.opcua_binary;title:Zeek OPC UA Binary logs;fields:zeek.opcua_binary.filter_source_link_id,zeek.opcua_binary.operand_source_link_id,zeek.opcua_binary.variant_source_link_id,zeek.opcua_binary.encoding_mask,zeek.opcua_binary.endpoint_url,zeek.opcua_binary.error,zeek.opcua_binary.identifier,zeek.opcua_binary.identifier_str,zeek.opcua_binary.is_final,zeek.opcua_binary.max_chunk_cnt,zeek.opcua_binary.max_msg_size,zeek.opcua_binary.msg_size,zeek.opcua_binary.msg_type,zeek.opcua_binary.namespace_idx,zeek.opcua_binary.opcua_link_id,zeek.opcua_binary.rcv_buf_size,zeek.opcua_binary.rcv_cert,zeek.opcua_binary.rcv_cert_len,zeek.opcua_binary.reason,zeek.opcua_binary.req_hdr_add_hdr_enc_mask,zeek.opcua_binary.req_hdr_add_hdr_type_id,zeek.opcua_binary.req_hdr_audit_entry_id,zeek.opcua_binary.req_hdr_node_id_guid,zeek.opcua_binary.req_hdr_node_id_namespace_idx,zeek.opcua_binary.req_hdr_node_id_numeric,zeek.opcua_binary.req_hdr_node_id_opaque,zeek.opcua_binary.req_hdr_node_id_string,zeek.opcua_binary.req_hdr_node_id_type,zeek.opcua_binary.req_hdr_request_handle,zeek.opcua_binary.req_hdr_return_diag,zeek.opcua_binary.req_hdr_timeout_hint,zeek.opcua_binary.req_hdr_timestamp,zeek.opcua_binary.request_id,zeek.opcua_binary.res_hdr_add_hdr_enc_mask,zeek.opcua_binary.res_hdr_add_hdr_type_id,zeek.opcua_binary.res_hdr_request_handle,zeek.opcua_binary.res_hdr_service_diag_encoding,zeek.opcua_binary.res_hdr_timestamp,zeek.opcua_binary.sec_channel_id,zeek.opcua_binary.sec_policy_uri,zeek.opcua_binary.sec_policy_uri_len,zeek.opcua_binary.seq_number,zeek.opcua_binary.snd_buf_size,zeek.opcua_binary.snd_cert,zeek.opcua_binary.snd_cert_len,zeek.opcua_binary.version,zeek.opcua_binary_activate_session.client_algorithm,zeek.opcua_binary_activate_session.client_signature,zeek.opcua_binary_activate_session.ext_obj_certificate_data,zeek.opcua_binary_activate_session.ext_obj_encoding,zeek.opcua_binary_activate_session.ext_obj_encryption_algorithom,zeek.opcua_binary_activate_session.ext_obj_password,zeek.opcua_binary_activate_session.ext_obj_policy_id,zeek.opcua_binary_activate_session.ext_obj_token_data,zeek.opcua_binary_activate_session.ext_obj_type_id_encoding_mask,zeek.opcua_binary_activate_session.ext_obj_type_id_guid,zeek.opcua_binary_activate_session.ext_obj_type_id_namespace_idx,zeek.opcua_binary_activate_session.ext_obj_type_id_numeric,zeek.opcua_binary_activate_session.ext_obj_type_id_opaque,zeek.opcua_binary_activate_session.ext_obj_type_id_str,zeek.opcua_binary_activate_session.ext_obj_type_id_string,zeek.opcua_binary_activate_session.ext_obj_user_name,zeek.opcua_binary_activate_session.server_nonce,zeek.opcua_binary_activate_session.user_token_algorithm,zeek.opcua_binary_activate_session.user_token_signature,zeek.opcua_binary_activate_session_client_software_cert.cert_data,zeek.opcua_binary_activate_session_client_software_cert.cert_signature,zeek.opcua_binary_activate_session_client_software_cert.client_software_cert_link_id,zeek.opcua_binary_activate_session_locale_id.local_id,zeek.opcua_binary_activate_session_locale_id.opcua_locale_link_id,zeek.opcua_binary_aggregate_filter.aggregate_type_encoding_mask,zeek.opcua_binary_aggregate_filter.aggregate_type_guid,zeek.opcua_binary_aggregate_filter.aggregate_type_namespace_idx,zeek.opcua_binary_aggregate_filter.aggregate_type_numeric,zeek.opcua_binary_aggregate_filter.aggregate_type_opaque,zeek.opcua_binary_aggregate_filter.aggregate_type_string,zeek.opcua_binary_aggregate_filter.percent_data_bad,zeek.opcua_binary_aggregate_filter.percent_data_good,zeek.opcua_binary_aggregate_filter.processing_interval,zeek.opcua_binary_aggregate_filter.revised_percent_data_bad,zeek.opcua_binary_aggregate_filter.revised_percent_data_good,zeek.opcua_binary_aggregate_filter.revised_processing_interval,zeek.opcua_binary_aggregate_filter.revised_start_time,zeek.opcua_binary_aggregate_filter.revised_start_time_str,zeek.opcua_binary_aggregate_filter.revised_treat_uncertain_as_bad,zeek.opcua_binary_aggregate_filter.revised_use_server_capabilities_default,zeek.opcua_binary_aggregate_filter.revised_use_slopped_extrapolation,zeek.opcua_binary_aggregate_filter.start_time,zeek.opcua_binary_aggregate_filter.start_time_str,zeek.opcua_binary_aggregate_filter.treat_uncertain_as_bad,zeek.opcua_binary_aggregate_filter.use_server_capabilities_default,zeek.opcua_binary_aggregate_filter.use_slopped_extrapolation,zeek.opcua_binary_browse.browse_next_release_continuation_point,zeek.opcua_binary_browse.browse_service_type,zeek.opcua_binary_browse.browse_view_description_timestamp,zeek.opcua_binary_browse.browse_view_description_view_version,zeek.opcua_binary_browse.browse_view_id_encoding_mask,zeek.opcua_binary_browse.browse_view_id_guid,zeek.opcua_binary_browse.browse_view_id_namespace_idx,zeek.opcua_binary_browse.browse_view_id_numeric,zeek.opcua_binary_browse.browse_view_id_opaque,zeek.opcua_binary_browse.browse_view_id_string,zeek.opcua_binary_browse.req_max_ref_nodes,zeek.opcua_binary_browse_description.browse_description_encoding_mask,zeek.opcua_binary_browse_description.browse_description_guid,zeek.opcua_binary_browse_description.browse_description_include_subtypes,zeek.opcua_binary_browse_description.browse_description_link_id,zeek.opcua_binary_browse_description.browse_description_namespace_idx,zeek.opcua_binary_browse_description.browse_description_numeric,zeek.opcua_binary_browse_description.browse_description_opaque,zeek.opcua_binary_browse_description.browse_description_ref_encoding_mask,zeek.opcua_binary_browse_description.browse_description_ref_guid,zeek.opcua_binary_browse_description.browse_description_ref_namespace_idx,zeek.opcua_binary_browse_description.browse_description_ref_numeric,zeek.opcua_binary_browse_description.browse_description_ref_opaque,zeek.opcua_binary_browse_description.browse_description_ref_string,zeek.opcua_binary_browse_description.browse_description_string,zeek.opcua_binary_browse_description.browse_direction,zeek.opcua_binary_browse_description.browse_node_class_mask,zeek.opcua_binary_browse_description.browse_result_mask,zeek.opcua_binary_browse_request_continuation_point.browse_next_link_id,zeek.opcua_binary_browse_request_continuation_point.continuation_point,zeek.opcua_binary_browse_response_references.browse_reference_link_id,zeek.opcua_binary_browse_response_references.browse_response_display_name_locale,zeek.opcua_binary_browse_response_references.browse_response_display_name_mask,zeek.opcua_binary_browse_response_references.browse_response_display_name_text,zeek.opcua_binary_browse_response_references.browse_response_is_forward,zeek.opcua_binary_browse_response_references.browse_response_node_class,zeek.opcua_binary_browse_response_references.browse_response_ref_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_ref_guid,zeek.opcua_binary_browse_response_references.browse_response_ref_name,zeek.opcua_binary_browse_response_references.browse_response_ref_name_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_numeric,zeek.opcua_binary_browse_response_references.browse_response_ref_opaque,zeek.opcua_binary_browse_response_references.browse_response_ref_string,zeek.opcua_binary_browse_response_references.browse_response_ref_type_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_ref_type_guid,zeek.opcua_binary_browse_response_references.browse_response_ref_type_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_type_namespace_uri,zeek.opcua_binary_browse_response_references.browse_response_ref_type_numeric,zeek.opcua_binary_browse_response_references.browse_response_ref_type_opaque,zeek.opcua_binary_browse_response_references.browse_response_ref_type_server_idx,zeek.opcua_binary_browse_response_references.browse_response_ref_type_string,zeek.opcua_binary_browse_response_references.browse_response_type_def_encoding_mask,zeek.opcua_binary_browse_response_references.browse_response_type_def_guid,zeek.opcua_binary_browse_response_references.browse_response_type_def_namespace_idx,zeek.opcua_binary_browse_response_references.browse_response_type_def_namespace_uri,zeek.opcua_binary_browse_response_references.browse_response_type_def_numeric,zeek.opcua_binary_browse_response_references.browse_response_type_def_opaque,zeek.opcua_binary_browse_response_references.browse_response_type_def_server_idx,zeek.opcua_binary_browse_response_references.browse_response_type_def_string,zeek.opcua_binary_browse_result.browse_response_link_id,zeek.opcua_binary_browse_result.browse_result_continuation_point,zeek.opcua_binary_close_session.del_subscriptions,zeek.opcua_binary_create_monitored_items.subscription_id,zeek.opcua_binary_create_monitored_items.timestamps_to_return,zeek.opcua_binary_create_monitored_items.timestamps_to_return_str,zeek.opcua_binary_create_monitored_items_create_item.create_item_link_id,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_attribute_id,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_index_range,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_name,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_encoding_mask,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_guid,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_numeric,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_opaque,zeek.opcua_binary_create_monitored_items_create_item.item_to_monitor_node_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitored_item_index_id,zeek.opcua_binary_create_monitored_items_create_item.monitoring_mode,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_client_handle,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_discard_oldest,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_encoding,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_encoding_mask,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_guid,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_namespace_idx,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_numeric,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_opaque,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_node_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_filter_info_type_id_string,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_queue_size,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_revised_queue_size,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_revised_sampling_interval,zeek.opcua_binary_create_monitored_items_create_item.monitoring_parameters_sampling_interval,zeek.opcua_binary_create_session.algorithm,zeek.opcua_binary_create_session.application_type,zeek.opcua_binary_create_session.application_uri,zeek.opcua_binary_create_session.auth_token_encoding_mask,zeek.opcua_binary_create_session.auth_token_guid,zeek.opcua_binary_create_session.auth_token_namespace_idx,zeek.opcua_binary_create_session.auth_token_numeric,zeek.opcua_binary_create_session.auth_token_opaque,zeek.opcua_binary_create_session.auth_token_string,zeek.opcua_binary_create_session.client_cert,zeek.opcua_binary_create_session.client_cert_size,zeek.opcua_binary_create_session.client_nonce,zeek.opcua_binary_create_session.discovery_profile_uri,zeek.opcua_binary_create_session.encoding_mask,zeek.opcua_binary_create_session.endpoint_url,zeek.opcua_binary_create_session.gateway_server_uri,zeek.opcua_binary_create_session.locale,zeek.opcua_binary_create_session.max_req_msg_size,zeek.opcua_binary_create_session.max_res_msg_size,zeek.opcua_binary_create_session.product_uri,zeek.opcua_binary_create_session.req_session_timeout,zeek.opcua_binary_create_session.revised_session_timeout,zeek.opcua_binary_create_session.server_cert,zeek.opcua_binary_create_session.server_cert_size,zeek.opcua_binary_create_session.server_nonce,zeek.opcua_binary_create_session.server_uri,zeek.opcua_binary_create_session.session_id_encoding_mask,zeek.opcua_binary_create_session.session_id_guid,zeek.opcua_binary_create_session.session_id_namespace_idx,zeek.opcua_binary_create_session.session_id_numeric,zeek.opcua_binary_create_session.session_id_opaque,zeek.opcua_binary_create_session.session_id_string,zeek.opcua_binary_create_session.session_name,zeek.opcua_binary_create_session.signature,zeek.opcua_binary_create_session.text,zeek.opcua_binary_create_session_discovery.discovery_profile_link_id,zeek.opcua_binary_create_session_discovery.discovery_profile_uri,zeek.opcua_binary_create_session_discovery.discovery_profile_url,zeek.opcua_binary_create_session_endpoints.application_type,zeek.opcua_binary_create_session_endpoints.application_uri,zeek.opcua_binary_create_session_endpoints.cert_size,zeek.opcua_binary_create_session_endpoints.discovery_profile_uri,zeek.opcua_binary_create_session_endpoints.encoding_mask,zeek.opcua_binary_create_session_endpoints.endpoint_link_id,zeek.opcua_binary_create_session_endpoints.endpoint_url,zeek.opcua_binary_create_session_endpoints.gateway_server_uri,zeek.opcua_binary_create_session_endpoints.locale,zeek.opcua_binary_create_session_endpoints.message_security_mode,zeek.opcua_binary_create_session_endpoints.product_uri,zeek.opcua_binary_create_session_endpoints.security_level,zeek.opcua_binary_create_session_endpoints.security_policy_uri,zeek.opcua_binary_create_session_endpoints.server_cert,zeek.opcua_binary_create_session_endpoints.text,zeek.opcua_binary_create_session_endpoints.transport_profile_uri,zeek.opcua_binary_create_session_user_token.user_token_endpoint_url,zeek.opcua_binary_create_session_user_token.user_token_issued_type,zeek.opcua_binary_create_session_user_token.user_token_link_id,zeek.opcua_binary_create_session_user_token.user_token_policy_id,zeek.opcua_binary_create_session_user_token.user_token_sec_policy_uri,zeek.opcua_binary_create_session_user_token.user_token_type,zeek.opcua_binary_create_subscription.max_notifications_per_publish,zeek.opcua_binary_create_subscription.priority,zeek.opcua_binary_create_subscription.publishing_enabled,zeek.opcua_binary_create_subscription.requested_lifetime_count,zeek.opcua_binary_create_subscription.requested_max_keep_alive_count,zeek.opcua_binary_create_subscription.requested_publishing_interval,zeek.opcua_binary_create_subscription.revised_lifetime_count,zeek.opcua_binary_create_subscription.revised_max_keep_alive_count,zeek.opcua_binary_create_subscription.revised_publishing_interval,zeek.opcua_binary_create_subscription.subscription_id,zeek.opcua_binary_data_change_filter.deadband_type,zeek.opcua_binary_data_change_filter.deadband_value,zeek.opcua_binary_data_change_filter.trigger,zeek.opcua_binary_diag_info_detail.addl_info,zeek.opcua_binary_diag_info_detail.diag_info_link_id,zeek.opcua_binary_diag_info_detail.has_addl_info,zeek.opcua_binary_diag_info_detail.has_inner_diag_info,zeek.opcua_binary_diag_info_detail.has_inner_stat_code,zeek.opcua_binary_diag_info_detail.has_locale,zeek.opcua_binary_diag_info_detail.has_locale_txt,zeek.opcua_binary_diag_info_detail.has_namespace_uri,zeek.opcua_binary_diag_info_detail.has_symbolic_id,zeek.opcua_binary_diag_info_detail.inner_diag_level,zeek.opcua_binary_diag_info_detail.inner_stat_code,zeek.opcua_binary_diag_info_detail.locale,zeek.opcua_binary_diag_info_detail.locale_str,zeek.opcua_binary_diag_info_detail.locale_txt,zeek.opcua_binary_diag_info_detail.locale_txt_str,zeek.opcua_binary_diag_info_detail.namespace_uri,zeek.opcua_binary_diag_info_detail.namespace_uri_str,zeek.opcua_binary_diag_info_detail.root_object_id,zeek.opcua_binary_diag_info_detail.source,zeek.opcua_binary_diag_info_detail.source_str,zeek.opcua_binary_diag_info_detail.symbolic_id,zeek.opcua_binary_diag_info_detail.symbolic_id_str,zeek.opcua_binary_event_filter_attribute_operand.alias,zeek.opcua_binary_event_filter_attribute_operand.attribute,zeek.opcua_binary_event_filter_attribute_operand.index_range,zeek.opcua_binary_event_filter_attribute_operand.node_id_encoding_mask,zeek.opcua_binary_event_filter_attribute_operand.node_id_guid,zeek.opcua_binary_event_filter_attribute_operand.node_id_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand.node_id_numeric,zeek.opcua_binary_event_filter_attribute_operand.node_id_opaque,zeek.opcua_binary_event_filter_attribute_operand.node_id_string,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.browse_path_element_link_id,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.include_subtypes,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.is_inverse,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.target_name,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.target_name_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_encoding_mask,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_guid,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_namespace_idx,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_numeric,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_opaque,zeek.opcua_binary_event_filter_attribute_operand_browse_paths.type_id_string,zeek.opcua_binary_event_filter_element_operand.element_index,zeek.opcua_binary_event_filter_select_clause.attribute_id,zeek.opcua_binary_event_filter_select_clause.index_range,zeek.opcua_binary_event_filter_select_clause.select_clause_link_id,zeek.opcua_binary_event_filter_select_clause.type_id_encoding_mask,zeek.opcua_binary_event_filter_select_clause.type_id_guid,zeek.opcua_binary_event_filter_select_clause.type_id_namespace_idx,zeek.opcua_binary_event_filter_select_clause.type_id_numeric,zeek.opcua_binary_event_filter_select_clause.type_id_opaque,zeek.opcua_binary_event_filter_select_clause.type_id_string,zeek.opcua_binary_event_filter_simple_attribute_operand.attribute_id,zeek.opcua_binary_event_filter_simple_attribute_operand.index_range,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_encoding_mask,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_guid,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_namespace_idx,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_numeric,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_opaque,zeek.opcua_binary_event_filter_simple_attribute_operand.type_id_string,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.browse_path_src,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.name,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.namespace_index,zeek.opcua_binary_event_filter_simple_attribute_operand_browse_paths.simple_attribute_operand_browse_path_link_id,zeek.opcua_binary_event_filter_where_clause.where_clause_link_id,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_element_link_id,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_encoding,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_encoding_mask,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_guid,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_namespace_idx,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_numeric,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_opaque,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_node_id_string,zeek.opcua_binary_event_filter_where_clause_elements.content_filter_filter_operand_type_id_string,zeek.opcua_binary_event_filter_where_clause_elements.filter_operator,zeek.opcua_binary_get_endpoints.endpoint_url,zeek.opcua_binary_get_endpoints_description.application_type,zeek.opcua_binary_get_endpoints_description.application_uri,zeek.opcua_binary_get_endpoints_description.cert_size,zeek.opcua_binary_get_endpoints_description.discovery_profile_uri,zeek.opcua_binary_get_endpoints_description.encoding_mask,zeek.opcua_binary_get_endpoints_description.endpoint_description_link_id,zeek.opcua_binary_get_endpoints_description.endpoint_uri,zeek.opcua_binary_get_endpoints_description.gateway_server_uri,zeek.opcua_binary_get_endpoints_description.locale,zeek.opcua_binary_get_endpoints_description.message_security_mode,zeek.opcua_binary_get_endpoints_description.product_uri,zeek.opcua_binary_get_endpoints_description.security_level,zeek.opcua_binary_get_endpoints_description.security_policy_uri,zeek.opcua_binary_get_endpoints_description.server_cert,zeek.opcua_binary_get_endpoints_description.text,zeek.opcua_binary_get_endpoints_description.transport_profile_uri,zeek.opcua_binary_get_endpoints_discovery.discovery_profile_link_id,zeek.opcua_binary_get_endpoints_discovery.discovery_profile_url,zeek.opcua_binary_get_endpoints_locale_id.locale_id,zeek.opcua_binary_get_endpoints_locale_id.locale_link_id,zeek.opcua_binary_get_endpoints_profile_uri.profile_uri,zeek.opcua_binary_get_endpoints_profile_uri.profile_uri_link_id,zeek.opcua_binary_get_endpoints_user_token.user_token_endpoint_url,zeek.opcua_binary_get_endpoints_user_token.user_token_issued_type,zeek.opcua_binary_get_endpoints_user_token.user_token_link_id,zeek.opcua_binary_get_endpoints_user_token.user_token_policy_id,zeek.opcua_binary_get_endpoints_user_token.user_token_sec_policy_uri,zeek.opcua_binary_get_endpoints_user_token.user_token_type,zeek.opcua_binary_opensecure_channel.client_nonce,zeek.opcua_binary_opensecure_channel.client_proto_ver,zeek.opcua_binary_opensecure_channel.message_security_mode,zeek.opcua_binary_opensecure_channel.req_lifetime,zeek.opcua_binary_opensecure_channel.sec_token_created_at,zeek.opcua_binary_opensecure_channel.sec_token_id,zeek.opcua_binary_opensecure_channel.sec_token_request_type,zeek.opcua_binary_opensecure_channel.sec_token_revised_time,zeek.opcua_binary_opensecure_channel.sec_token_sec_channel_id,zeek.opcua_binary_opensecure_channel.server_nonce,zeek.opcua_binary_opensecure_channel.server_proto_ver,zeek.opcua_binary_read.max_age,zeek.opcua_binary_read.timestamps_to_return,zeek.opcua_binary_read.timestamps_to_return_str,zeek.opcua_binary_read_nodes_to_read.attribute_id,zeek.opcua_binary_read_nodes_to_read.attribute_id_str,zeek.opcua_binary_read_nodes_to_read.data_encoding_name,zeek.opcua_binary_read_nodes_to_read.data_encoding_name_idx,zeek.opcua_binary_read_nodes_to_read.index_range,zeek.opcua_binary_read_nodes_to_read.node_id_encoding_mask,zeek.opcua_binary_read_nodes_to_read.node_id_guid,zeek.opcua_binary_read_nodes_to_read.node_id_namespace_idx,zeek.opcua_binary_read_nodes_to_read.node_id_numeric,zeek.opcua_binary_read_nodes_to_read.node_id_opaque,zeek.opcua_binary_read_nodes_to_read.node_id_string,zeek.opcua_binary_read_nodes_to_read.nodes_to_read_link_id,zeek.opcua_binary_read_results.data_value_encoding_mask,zeek.opcua_binary_read_results.level,zeek.opcua_binary_read_results.results_link_id,zeek.opcua_binary_read_results.server_pico_sec,zeek.opcua_binary_read_results.server_timestamp,zeek.opcua_binary_read_results.source_pico_sec,zeek.opcua_binary_read_results.source_timestamp,zeek.opcua_binary_status_code_detail.historian_bits,zeek.opcua_binary_status_code_detail.historian_bits_str,zeek.opcua_binary_status_code_detail.historianextradata,zeek.opcua_binary_status_code_detail.historianmultivalue,zeek.opcua_binary_status_code_detail.historianpartial,zeek.opcua_binary_status_code_detail.info_type,zeek.opcua_binary_status_code_detail.info_type_str,zeek.opcua_binary_status_code_detail.limit_bits,zeek.opcua_binary_status_code_detail.limit_bits_str,zeek.opcua_binary_status_code_detail.overflow,zeek.opcua_binary_status_code_detail.semantics_changed,zeek.opcua_binary_status_code_detail.severity,zeek.opcua_binary_status_code_detail.severity_str,zeek.opcua_binary_status_code_detail.source,zeek.opcua_binary_status_code_detail.source_level,zeek.opcua_binary_status_code_detail.source_str,zeek.opcua_binary_status_code_detail.status_code,zeek.opcua_binary_status_code_detail.status_code_link_id,zeek.opcua_binary_status_code_detail.structure_changed,zeek.opcua_binary_status_code_detail.sub_code,zeek.opcua_binary_status_code_detail.sub_code_str,zeek.opcua_binary_variant_array_dims.array_dim_link_id,zeek.opcua_binary_variant_array_dims.dimension,zeek.opcua_binary_variant_data.variant_data_encoding_name,zeek.opcua_binary_variant_data.variant_data_encoding_name_idx,zeek.opcua_binary_variant_data.variant_data_link_id,zeek.opcua_binary_variant_data.variant_data_locale,zeek.opcua_binary_variant_data.variant_data_mask,zeek.opcua_binary_variant_data.variant_data_node_id_encoding_mask,zeek.opcua_binary_variant_data.variant_data_node_id_guid,zeek.opcua_binary_variant_data.variant_data_node_id_namespace_idx,zeek.opcua_binary_variant_data.variant_data_node_id_namespace_uri,zeek.opcua_binary_variant_data.variant_data_node_id_numeric,zeek.opcua_binary_variant_data.variant_data_node_id_opaque,zeek.opcua_binary_variant_data.variant_data_node_id_server_idx,zeek.opcua_binary_variant_data.variant_data_node_id_string,zeek.opcua_binary_variant_data.variant_data_text,zeek.opcua_binary_variant_data.variant_data_value_decimal,zeek.opcua_binary_variant_data.variant_data_value_signed_numeric,zeek.opcua_binary_variant_data.variant_data_value_string,zeek.opcua_binary_variant_data.variant_data_value_time,zeek.opcua_binary_variant_data.variant_data_value_unsigned_numeric,zeek.opcua_binary_variant_data_value.data_value_encoding_mask,zeek.opcua_binary_variant_data_value.server_pico_sec,zeek.opcua_binary_variant_data_value.server_timestamp,zeek.opcua_binary_variant_data_value.source_pico_sec,zeek.opcua_binary_variant_data_value.source_timestamp,zeek.opcua_binary_variant_data_value.variant_data_value_source_link,zeek.opcua_binary_variant_extension_object.ext_obj_encoding,zeek.opcua_binary_variant_extension_object.ext_obj_link_id,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_encoding_mask,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_guid,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_namespace_idx,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_numeric,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_opaque,zeek.opcua_binary_variant_extension_object.ext_obj_node_id_string,zeek.opcua_binary_variant_extension_object.ext_obj_type_id_str,zeek.opcua_binary_variant_metadata.built_in_data_type,zeek.opcua_binary_variant_metadata.built_in_data_type_str,zeek.opcua_binary_variant_metadata.dara_variant_encoding_mask,zeek.opcua_binary_variant_metadata.data_variant_data_type,zeek.opcua_binary_variant_metadata.data_variant_data_type_str,zeek.opcua_binary_variant_metadata.variant_data_array_dim,zeek.opcua_binary_variant_metadata.variant_data_source,zeek.opcua_binary_variant_metadata.variant_data_source_str,zeek_opcua_binary_write=require:zeek.opcua_binary_write;title:Zeek opcua_binary_write.log;fields:zeek.opcua_binary_write.source_h,zeek.opcua_binary_write.source_p,zeek.opcua_binary_write.destination_h,zeek.opcua_binary_write.destination_p,zeek.opcua_binary_write.node_id_encoding_mask,zeek.opcua_binary_write.node_id_namespace_idx,zeek.opcua_binary_write.node_id_numeric,zeek.opcua_binary_write.node_id_string,zeek.opcua_binary_write.node_id_guid,zeek.opcua_binary_write.node_id_opaque,zeek.opcua_binary_write.attribute_id,zeek.opcua_binary_write.attribute_id_str,zeek.opcua_binary_write.index_range,zeek.opcua_binary_write.data_value_encoding_mask,zeek.opcua_binary_write.source_timestamp,zeek.opcua_binary_write.source_pico_sec,zeek.opcua_binary_write.server_timestamp,zeek.opcua_binary_write.server_pico_sec o_zeek_ospf=require:zeek.ospf;title:Zeek ospf.log;fields:zeek.ospf.ospf_type,zeek.ospf.version,zeek.ospf.router_id,zeek.ospf.area_id,zeek.ospf.interface_id,zeek.ospf.netmask,zeek.ospf.desig_router,zeek.ospf.backup_router,zeek.ospf.neighbors,zeek.ospf.lsa_type,zeek.ospf.link_state_id,zeek.ospf.advert_router,zeek.ospf.routers,zeek.ospf.link_id,zeek.ospf.link_data,zeek.ospf.link_type,zeek.ospf.neighbor_router_id,zeek.ospf.metrics,zeek.ospf.fwd_addrs,zeek.ospf.route_tags,zeek.ospf.neighbor_interface_id,zeek.ospf.prefix,zeek.ospf.metric,zeek.ospf.dest_router_id,zeek.ospf.link_prefixes,zeek.ospf.intra_prefixes o_zeek_pe=require:zeek.pe;title:Zeek pe.log;fields:zeek.pe.machine,zeek.pe.compile_ts,zeek.pe.os,zeek.pe.subsystem,zeek.pe.is_exe,zeek.pe.is_64bit,zeek.pe.uses_aslr,zeek.pe.uses_dep,zeek.pe.uses_code_integrity,zeek.pe.uses_seh,zeek.pe.has_import_table,zeek.pe.has_export_table,zeek.pe.has_cert_table,zeek.pe.has_debug_data,zeek.pe.section_names o_zeek_profinet=require:zeek.profinet;title:Zeek profinet.log;fields:zeek.profinet.operation_type,zeek.profinet.block_version,zeek.profinet.slot_number,zeek.profinet.subslot_number,zeek.profinet.index diff --git a/arkime/wise/source.zeeklogs.js b/arkime/wise/source.zeeklogs.js index 33e520ca9..2bc468f3f 100644 --- a/arkime/wise/source.zeeklogs.js +++ b/arkime/wise/source.zeeklogs.js @@ -1666,19 +1666,10 @@ class MalcolmSource extends WISESource { "zeek.http.user_agent", "zeek.http.version", "zeek.intel.cif_confidence", - "zeek.intel.cif_description", - "zeek.intel.cif_firstseen", - "zeek.intel.cif_lastseen", - "zeek.intel.cif_source", - "zeek.intel.cif_tags", "zeek.intel.file_description", "zeek.intel.file_mime_type", - "zeek.intel.matched", - "zeek.intel.seen_indicator", - "zeek.intel.seen_indicator_type", "zeek.intel.seen_node", "zeek.intel.seen_where", - "zeek.intel.sources", "zeek.ipsec.certificates", "zeek.ipsec.doi", "zeek.ipsec.exchange_type", @@ -2304,6 +2295,20 @@ class MalcolmSource extends WISESource { "zeek.opcua_binary_variant_metadata.variant_data_array_dim", "zeek.opcua_binary_variant_metadata.variant_data_source", "zeek.opcua_binary_variant_metadata.variant_data_source_str", + "zeek.opcua_binary_write.node_id_encoding_mask", + "zeek.opcua_binary_write.node_id_namespace_idx", + "zeek.opcua_binary_write.node_id_numeric", + "zeek.opcua_binary_write.node_id_string", + "zeek.opcua_binary_write.node_id_guid", + "zeek.opcua_binary_write.node_id_opaque", + "zeek.opcua_binary_write.attribute_id", + "zeek.opcua_binary_write.attribute_id_str", + "zeek.opcua_binary_write.index_range", + "zeek.opcua_binary_write.data_value_encoding_mask", + "zeek.opcua_binary_write.source_timestamp", + "zeek.opcua_binary_write.source_pico_sec", + "zeek.opcua_binary_write.server_timestamp", + "zeek.opcua_binary_write.server_pico_sec", "zeek.ospf.advert_router", "zeek.ospf.area_id", "zeek.ospf.backup_router", diff --git a/config/logstash.env.example b/config/logstash.env.example index 3db494a47..7f2682043 100644 --- a/config/logstash.env.example +++ b/config/logstash.env.example @@ -14,4 +14,4 @@ LOGSTASH_NETBOX_ENRICHMENT_DATASETS=suricata.alert,zeek.conn,zeek.dhcp,zeek.dns, # Zeek log types that will be ignored (dropped) by LogStash LOGSTASH_ZEEK_IGNORED_LOGS=analyzer,broker,cluster,config,loaded_scripts,packet_filter,png,print,prof,reporter,stderr,stdout # Logstash memory allowance and other Java options -LS_JAVA_OPTS=-server -Xmx2500m -Xms2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true \ No newline at end of file +LS_JAVA_OPTS=-server -Xmx2500m -Xms2500m -Xss1536k -XX:-HeapDumpOnOutOfMemoryError -Djava.security.egd=file:/dev/./urandom -Dlog4j.formatMsgNoLookups=true -Dlogstash.pipelinebus.implementation=v1 \ No newline at end of file diff --git a/config/zeek-offline.env.example b/config/zeek-offline.env.example index 75c2e130a..6e4efc7c5 100644 --- a/config/zeek-offline.env.example +++ b/config/zeek-offline.env.example @@ -11,9 +11,9 @@ ZEEK_ROTATED_PCAP=true ZEEK_PCAP_PROCESSOR=true -# Specifies whether or not to refresh Zeek Intelligence Framework files in -# the container entrypoint -ZEEK_INTEL_REFRESH_ON_ENTRYPOINT=true +# Specifies whether or not to refresh Zeek Intelligence Framework files +# as soon as the container starts up +ZEEK_INTEL_REFRESH_ON_STARTUP=true # Specifies a cron expression indicating the refresh interval for generating the # Zeek Intelligence Framework files (or blank to disable automatic refresh) ZEEK_INTEL_REFRESH_CRON_EXPRESSION= \ No newline at end of file diff --git a/config/zeek.env.example b/config/zeek.env.example index 6434eba1a..5c69531bf 100644 --- a/config/zeek.env.example +++ b/config/zeek.env.example @@ -7,11 +7,11 @@ ZEEK_LOCAL_NETS= ZEEK_JSON= # Specifies the value for Zeek's Intel::item_expiration timeout (-1min to disable) ZEEK_INTEL_ITEM_EXPIRATION=-1min -# When querying a TAXII or MISP feed, only process threat indicators that have +# When querying a threat intelligence feed, only process threat indicators that have # been created or modified since the time represented by this value; # it may be either a fixed date/time (01/01/2021) or relative interval (30 days ago) ZEEK_INTEL_FEED_SINCE= -# Whether or not to require SSL certificate verification when querying a TAXII or MISP feed +# Whether or not to require SSL certificate verification when querying an intelligence feed ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION=false # Number of threads to use for querying feeds for generating Zeek Intelligence Framework files ZEEK_INTEL_REFRESH_THREADS=2 @@ -99,3 +99,6 @@ ZEEK_SYNCHROPHASOR_DETAILED= ZEEK_GENISYS_PORTS= ZEEK_ENIP_PORTS= ZEEK_DISABLE_BEST_GUESS_ICS=true +ZEEK_KAFKA_ENABLED= +ZEEK_KAFKA_BROKERS=kafka.local:9091 +ZEEK_KAFKA_TOPIC=zeek \ No newline at end of file diff --git a/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json b/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json index d55123c06..cfd7012f1 100644 --- a/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json +++ b/dashboards/dashboards/36ed695f-edcc-47c1-b0ec-50d20c93ce0f.json @@ -1,5 +1,5 @@ { - "version": "1.2.0", + "version": "2.18.0", "objects": [ { "id": "36ed695f-edcc-47c1-b0ec-50d20c93ce0f", @@ -7,13 +7,13 @@ "namespaces": [ "default" ], - "updated_at": "2024-04-29T15:49:16.000Z", - "version": "WzEwMjMsMV0=", + "updated_at": "2024-11-12T16:22:49.507Z", + "version": "WzEwNTIsMV0=", "attributes": { "title": "Zeek Intelligence", "hits": 0, "description": "", - "panelsJSON": "[{\"embeddableConfig\":{},\"gridData\":{\"h\":26,\"i\":\"2\",\"w\":8,\"x\":0,\"y\":0},\"panelIndex\":\"2\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{\"table\":null,\"vis\":{\"legendOpen\":false}},\"gridData\":{\"h\":8,\"i\":\"3\",\"w\":32,\"x\":16,\"y\":0},\"panelIndex\":\"3\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"5\",\"w\":16,\"x\":8,\"y\":8},\"panelIndex\":\"5\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"6\",\"w\":15,\"x\":33,\"y\":26},\"panelIndex\":\"6\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"7\",\"w\":13,\"x\":14,\"y\":44},\"panelIndex\":\"7\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"8\",\"w\":21,\"x\":27,\"y\":44},\"panelIndex\":\"8\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"11\",\"w\":25,\"x\":8,\"y\":26},\"panelIndex\":\"11\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"12\",\"w\":14,\"x\":0,\"y\":44},\"panelIndex\":\"12\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"gridData\":{\"h\":18,\"i\":\"13\",\"w\":8,\"x\":0,\"y\":26},\"panelIndex\":\"13\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_8\"},{\"embeddableConfig\":{\"columns\":[\"source.ip\",\"destination.ip\",\"destination.port\",\"zeek.intel.seen_indicator\",\"zeek.intel.seen_indicator_type\",\"zeek.intel.sources\",\"zeek.intel.seen_where\",\"event.id\"],\"sort\":[\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"desc\"]},\"gridData\":{\"h\":24,\"i\":\"14\",\"w\":48,\"x\":0,\"y\":62},\"panelIndex\":\"14\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_9\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":18,\"i\":\"16\",\"w\":24,\"x\":24,\"y\":8},\"panelIndex\":\"16\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_10\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":8,\"i\":\"17\",\"w\":8,\"x\":8,\"y\":0},\"panelIndex\":\"17\",\"version\":\"1.2.0\",\"panelRefName\":\"panel_11\"}]", + "panelsJSON": "[{\"version\":\"2.18.0\",\"gridData\":{\"x\":0,\"y\":0,\"w\":8,\"h\":26,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":16,\"y\":0,\"w\":32,\"h\":8,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"table\":null,\"vis\":{\"legendOpen\":false}},\"panelRefName\":\"panel_1\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":16,\"y\":8,\"w\":10,\"h\":18,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":0,\"y\":26,\"w\":13,\"h\":18,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_3\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":13,\"y\":26,\"w\":11,\"h\":18,\"i\":\"7\"},\"panelIndex\":\"7\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_4\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":24,\"y\":26,\"w\":14,\"h\":18,\"i\":\"8\"},\"panelIndex\":\"8\",\"embeddableConfig\":{\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":2,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_5\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":26,\"y\":8,\"w\":22,\"h\":18,\"i\":\"11\"},\"panelIndex\":\"11\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":38,\"y\":26,\"w\":10,\"h\":18,\"i\":\"12\"},\"panelIndex\":\"12\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":8,\"y\":8,\"w\":8,\"h\":18,\"i\":\"13\"},\"panelIndex\":\"13\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":1,\"direction\":\"desc\"}}}},\"panelRefName\":\"panel_8\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":0,\"y\":44,\"w\":48,\"h\":24,\"i\":\"14\"},\"panelIndex\":\"14\",\"embeddableConfig\":{\"columns\":[\"source.ip\",\"destination.ip\",\"destination.port\",\"threat.indicator.name\",\"threat.indicator.type\",\"threat.indicator.provider\",\"zeek.intel.seen_where\",\"event.id\"],\"sort\":[]},\"panelRefName\":\"panel_9\"},{\"version\":\"2.18.0\",\"gridData\":{\"x\":8,\"y\":0,\"w\":8,\"h\":8,\"i\":\"17\"},\"panelIndex\":\"17\",\"embeddableConfig\":{},\"panelRefName\":\"panel_10\"}]", "optionsJSON": "{\"useMargins\":true}", "version": 1, "timeRestore": false, @@ -75,11 +75,6 @@ { "name": "panel_10", "type": "visualization", - "id": "fa56cc7f-fb00-47fb-becb-1b1fdfea908e" - }, - { - "name": "panel_11", - "type": "visualization", "id": "AWDG-Qf8xQT5EBNmq4G5" } ], @@ -93,8 +88,8 @@ "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:23:14.699Z", - "version": "Wzc2OSwxXQ==", + "updated_at": "2024-11-12T15:53:48.389Z", + "version": "WzkyOSwxXQ==", "attributes": { "title": "Navigation", "visState": "{\"title\":\"Navigation\",\"type\":\"markdown\",\"params\":{\"markdown\":\"### General Network Logs\\n[Overview](#/dashboard/0ad3d7c2-3441-485e-9dfe-dbb22e84e576) \\n[Security Overview](#/dashboard/95479950-41f2-11ea-88fa-7151df485405) \\n[ICS/IoT Security Overview](#/dashboard/4a4bde20-4760-11ea-949c-bbb5a9feecbf) \\n[Severity](#/dashboard/d2dd0180-06b1-11ec-8c6b-353266ade330) \\n[Connections](#/dashboard/abdd7550-2c7c-40dc-947e-f6d186a158c4) \\n[Actions and Results](#/dashboard/a33e0a50-afcd-11ea-993f-b7d8522a8bed) \\n[Files](#/dashboard/9ee51f94-3316-4fc5-bd89-93a52af69714) \\n[Executables](#/dashboard/0a490422-0ce9-44bf-9a2d-19329ddde8c3) \\n[Software](#/dashboard/87d990cc-9e0b-41e5-b8fe-b10ae1da0c85) \\n[Zeek Known Summary](#/dashboard/89d1cc50-974c-11ed-bb6b-3fb06c879b11) \\n[Zeek Intelligence](#/dashboard/36ed695f-edcc-47c1-b0ec-50d20c93ce0f) \\n[Zeek Notices](#/dashboard/f1f09567-fc7f-450b-a341-19d2f2bb468b) \\n[Zeek Weird](#/dashboard/1fff49f6-0199-4a0f-820b-721aff9ff1f1) \\n[Signatures](#/dashboard/665d1610-523d-11e9-a30e-e3576242f3ed) \\n[Suricata Alerts](#/dashboard/5694ca60-cbdf-11ec-a50a-5fedd672f5c5) \\n[Asset Interaction Analysis](#/dashboard/677ee170-809e-11ed-8d5b-07069f823b6f) \\n[↪ NetBox](/netbox/) \\n[↪ Arkime](/arkime/) \\n\\n### Common Protocols\\n[DCE/RPC](#/dashboard/432af556-c5c0-4cc3-8166-b274b4e3a406) ● [DHCP](#/dashboard/2d98bb8e-214c-4374-837b-20e1bcd63a5e) ● [DNS](#/dashboard/2cf94cd0-ecab-40a5-95a7-8419f3a39cd9) ● [FTP](#/dashboard/078b9aa5-9bd4-4f02-ae5e-cf80fa6f887b) / [TFTP](#/dashboard/bf5efbb0-60f1-11eb-9d60-dbf0411cfc48) ● [HTTP](#/dashboard/37041ee1-79c0-4684-a436-3173b0e89876) / [WebSocket](#/dashboard/b8cf5890-87ed-11ef-ae18-dbcd34795edb) ● [IRC](#/dashboard/76f2f912-80da-44cd-ab66-6a73c8344cc3) ● [Kerberos](#/dashboard/82da3101-2a9c-4ae2-bb61-d447a3fbe673) ● [LDAP](#/dashboard/05e3e000-f118-11e9-acda-83a8e29e1a24) ● [MQTT](#/dashboard/87a32f90-ef58-11e9-974e-9d600036d105) ● [MySQL](#/dashboard/50ced171-1b10-4c3f-8b67-2db9635661a6) ● [NTLM](#/dashboard/543118a9-02d7-43fe-b669-b8652177fc37) ● [NTP](#/dashboard/af5df620-eeb6-11e9-bdef-65a192b7f586) ● [OSPF](#/dashboard/1cc01ff0-5205-11ec-a62c-7bc80e88f3f0) ● [QUIC](#/dashboard/11ddd980-e388-11e9-b568-cf17de8e860c) ● [RADIUS](#/dashboard/ae79b7d1-4281-4095-b2f6-fa7eafda9970) ● [RDP](#/dashboard/7f41913f-cba8-43f5-82a8-241b7ead03e0) ● [RFB](#/dashboard/f77bf097-18a8-465c-b634-eb2acc7a4f26) ● [SIP](#/dashboard/0b2354ae-0fe9-4fd9-b156-1c3870e5c7aa) ● [SMB](#/dashboard/42e831b9-41a9-4f35-8b7d-e1566d368773) ● [SMTP](#/dashboard/bb827f8e-639e-468c-93c8-9f5bc132eb8f) ● [SNMP](#/dashboard/4e5f106e-c60a-4226-8f64-d534abb912ab) ● [SSH](#/dashboard/caef3ade-d289-4d05-a511-149f3e97f238) ● [SSL](#/dashboard/7f77b58a-df3e-4cc2-b782-fd7f8bad8ffb) / [X.509 Certificates](#/dashboard/024062a6-48d6-498f-a91a-3bf2da3a3cd3) ● [STUN](#/dashboard/fa477130-2b8a-11ec-a9f2-3911c8571bfd) ● [Syslog](#/dashboard/92985909-dc29-4533-9e80-d3182a0ecf1d) ● [TDS](#/dashboard/bed185a0-ef82-11e9-b38a-2db3ee640e88) / [TDS RPC](#/dashboard/32587740-ef88-11e9-b38a-2db3ee640e88) / [TDS SQL](#/dashboard/fa141950-ef89-11e9-b38a-2db3ee640e88) ● [Telnet / rlogin / rsh](#/dashboard/c2549e10-7f2e-11ea-9f8a-1fe1327e2cd2) ● [Tunnels](#/dashboard/11be6381-beef-40a7-bdce-88c5398392fc)\\n\\n### ICS/IoT Protocols\\n[BACnet](#/dashboard/2bec1490-eb94-11e9-a384-0fcf32210194) ● [BSAP](#/dashboard/ca5799a0-56b5-11eb-b749-576de068f8ad) ● [DNP3](#/dashboard/870a5862-6c26-4a08-99fd-0c06cda85ba3) ● [EtherCAT](#/dashboard/4a073440-b286-11eb-a4d4-09fa12a6ebd4) ● [EtherNet/IP](#/dashboard/29a1b290-eb98-11e9-a384-0fcf32210194) ● [GENISYS](#/dashboard/03207c00-d07e-11ec-b4a7-d1b4003706b7) ● [GE SRTP](#/dashboard/e233a570-45d9-11ef-96a6-432365601033) ● [HART-IP](#/dashboard/3a9e3440-75e2-11ef-8138-03748f839a49) ● [Modbus](#/dashboard/152f29dc-51a2-4f53-93e9-6e92765567b8) ● [OPCUA Binary](#/dashboard/dd87edd0-796a-11ec-9ce6-b395c1ff58f4) ● [PROFINET](#/dashboard/a7514350-eba6-11e9-a384-0fcf32210194) ● [S7comm](#/dashboard/e76d05c0-eb9f-11e9-a384-0fcf32210194) ● [Synchrophasor](#/dashboard/2cc56240-e460-11ed-a9d5-9f591c284cb4) ● [Best Guess](#/dashboard/12e3a130-d83b-11eb-a0b0-f328ce09b0b7)\\n\\n### Malcolm and Third-Party Logs\\n\\nResources: [System Overview](#/dashboard/Metricbeat-system-overview-ecs) / [Host Overview](#/dashboard/79ffd6e0-faa0-11e6-947f-177f697178b8-ecs) ● [Hardware Temperature](#/dashboard/0d4955f0-eb25-11ec-a6d4-b3526526c2c7) ● nginx [Overview](#/dashboard/55a9e6e0-a29e-11e7-928f-5dbe6f6f5519-ecs) / [Access and Error Logs](#/dashboard/046212a0-a2a1-11e7-928f-5dbe6f6f5519-ecs) ● Linux [Journald](#/dashboard/f6600310-9943-11ee-a029-e973f4774355) / [Kernel Messages](#/dashboard/3768ef70-d819-11ee-820d-dd9fd73a3921) ● [Windows Events](#/dashboard/79202ee0-d811-11ee-820d-dd9fd73a3921) ● [Malcolm Sensor File Integrity](#/dashboard/903f42c0-f634-11ec-828d-2fb7a4a26e1f) ● [Malcolm Sensor Audit Logs](#/dashboard/7a7e0a60-e8e8-11ec-b9d4-4569bb965430) ● [Packet Capture Statistics](#/dashboard/4ca94c70-d7da-11ee-9ed3-e7afff29e59a)\",\"type\":\"markdown\",\"fontSize\":10,\"openLinksInNewTab\":false},\"aggs\":[]}", @@ -116,8 +111,8 @@ "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:32:32.907Z", - "version": "Wzk5OCwxXQ==", + "updated_at": "2024-11-12T15:52:51.822Z", + "version": "WzI3MCwxXQ==", "attributes": { "title": "Intel - Log Count Over Time", "visState": "{\"title\":\"Intel - Log Count Over Time\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER\",\"timeRange\":{\"from\":\"now-25y\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\" \"},\"schema\":\"segment\"}],\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{\"text\":\"MALCOLM_NETWORK_INDEX_TIME_FIELD_REPLACER per 12 hours\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"\"}}],\"seriesParams\":[{\"show\":true,\"mode\":\"normal\",\"type\":\"histogram\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"showCircles\":true,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"type\":\"histogram\",\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", @@ -146,8 +141,8 @@ "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:22:26.156Z", - "version": "WzIzOCwxXQ==", + "updated_at": "2024-11-12T15:52:51.822Z", + "version": "WzI3MSwxXQ==", "attributes": { "title": "Intel - Seen", "visState": "{\"title\":\"Intel - Seen\",\"type\":\"pie\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_where\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":20,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Seen (Where)\"},\"schema\":\"segment\"}],\"params\":{\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"bottom\",\"isDonut\":true,\"type\":\"pie\",\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}}}", @@ -176,10 +171,10 @@ "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:22:26.156Z", - "version": "WzIzOSwxXQ==", + "updated_at": "2024-11-12T15:52:51.822Z", + "version": "WzI3MiwxXQ==", "attributes": { - "visState": "{\"title\":\"Intel - Source\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.intel.sources\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Source\"}}],\"listeners\":{}}", + "visState": "{\"title\":\"Intel - Source\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"threat.indicator.provider\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Source\"}}],\"listeners\":{}}", "description": "", "title": "Intel - Source", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", @@ -206,8 +201,8 @@ "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:22:26.156Z", - "version": "WzI0MCwxXQ==", + "updated_at": "2024-11-12T15:52:51.822Z", + "version": "WzI3MywxXQ==", "attributes": { "visState": "{\"title\":\"Intel - Source IP Address\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"source.ip\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"IP Address\"}}],\"listeners\":{}}", "description": "", @@ -236,8 +231,8 @@ "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:22:26.156Z", - "version": "WzI0MSwxXQ==", + "updated_at": "2024-11-12T15:52:51.822Z", + "version": "WzI3NCwxXQ==", "attributes": { "title": "Intel - Destination IP Address", "visState": "{\"title\":\"Intel - Destination IP Address\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.ip\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"IP Address\"},\"schema\":\"bucket\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"destination.port\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Port\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}", @@ -266,11 +261,11 @@ "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:22:26.156Z", - "version": "WzI0MiwxXQ==", + "updated_at": "2024-11-12T15:52:51.822Z", + "version": "WzI3NSwxXQ==", "attributes": { "title": "Intel - Indicator", - "visState": "{\"title\":\"Intel - Indicator\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_indicator_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_where\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Seen Where\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_indicator\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Indicator\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}", + "visState": "{\"title\":\"Intel - Indicator\",\"type\":\"table\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"threat.indicator.type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Type\"},\"schema\":\"bucket\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_where\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Seen Where\"},\"schema\":\"bucket\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"threat.indicator.name\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Indicator\"},\"schema\":\"bucket\"}],\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"showMetricsAtAllLevels\":false,\"percentageCol\":\"\"}}", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":3,\"direction\":\"desc\"}}}}", "description": "", "version": 1, @@ -296,8 +291,8 @@ "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:22:26.156Z", - "version": "WzI0MywxXQ==", + "updated_at": "2024-11-12T15:52:51.822Z", + "version": "WzI3NiwxXQ==", "attributes": { "visState": "{\"title\":\"Intel - MIME Type\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.intel.file_mime_type\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"MIME Type\"}}],\"listeners\":{}}", "description": "", @@ -326,12 +321,12 @@ "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:22:26.156Z", - "version": "WzI0NCwxXQ==", + "updated_at": "2024-11-12T15:52:51.822Z", + "version": "WzI3NywxXQ==", "attributes": { - "visState": "{\"title\":\"Intel - Matched\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"zeek.intel.matched\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Type Matched\"}}],\"listeners\":{}}", + "visState": "{\"title\":\"Intel - Indicator Type\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"threat.indicator.type\",\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Type Matched\"}}],\"listeners\":{}}", "description": "", - "title": "Intel - Matched", + "title": "Intel - Indicator Type", "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}", "version": 1, "kibanaSavedObjectMeta": { @@ -356,8 +351,8 @@ "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:22:26.156Z", - "version": "WzI0NSwxXQ==", + "updated_at": "2024-11-12T15:52:51.822Z", + "version": "WzI3OCwxXQ==", "attributes": { "sort": [ [ @@ -376,9 +371,9 @@ "source.ip", "destination.ip", "destination.port", - "zeek.intel.seen_indicator", - "zeek.intel.seen_indicator_type", - "zeek.intel.sources", + "threat.indicator.name", + "threat.indicator.type", + "threat.indicator.provider", "zeek.intel.seen_where", "event.id" ] @@ -394,44 +389,14 @@ "search": "7.9.3" } }, - { - "id": "fa56cc7f-fb00-47fb-becb-1b1fdfea908e", - "type": "visualization", - "namespaces": [ - "default" - ], - "updated_at": "2022-01-12T18:32:43.892Z", - "version": "WzEwMTIsMV0=", - "attributes": { - "title": "Intel - Indicator Type", - "visState": "{\"title\":\"Intel - Indicator Type\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"zeek.intel.seen_indicator_type\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"customLabel\":\"Indicator Type\"},\"schema\":\"segment\"}],\"params\":{\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"rotate\":0},\"title\":{\"text\":\"Indicator Type\"}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"type\":\"histogram\",\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}", - "uiStateJSON": "{\"vis\":{\"legendOpen\":false}}", - "description": "", - "version": 1, - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}" - }, - "savedSearchRefName": "search_0" - }, - "references": [ - { - "name": "search_0", - "type": "search", - "id": "5154d8e9-c83e-4d42-bde3-33ad0c7d1798" - } - ], - "migrationVersion": { - "visualization": "7.10.0" - } - }, { "id": "AWDG-Qf8xQT5EBNmq4G5", "type": "visualization", "namespaces": [ "default" ], - "updated_at": "2022-01-12T18:22:26.156Z", - "version": "WzI0NywxXQ==", + "updated_at": "2024-11-12T15:52:51.822Z", + "version": "WzI4MCwxXQ==", "attributes": { "title": "Intel - Log Count", "visState": "{\"title\":\"Intel - Log Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"None\",\"useRange\":false,\"colorsRange\":[{\"from\":0,\"to\":100}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":false,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"fontSize\":\"30\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"bgFill\":\"#FB9E00\"}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}", @@ -455,4 +420,4 @@ } } ] -} +} \ No newline at end of file diff --git a/dashboards/templates/composable/component/zeek.json b/dashboards/templates/composable/component/zeek.json index 83b43e30e..d4fae78d4 100644 --- a/dashboards/templates/composable/component/zeek.json +++ b/dashboards/templates/composable/component/zeek.json @@ -139,19 +139,10 @@ "zeek.http.user_agent": { "type": "keyword", "ignore_above": 256, "fields": { "text": { "type": "text" } } }, "zeek.http.version": { "type": "keyword" }, "zeek.intel.cif_confidence": { "type": "float" }, - "zeek.intel.cif_description": { "type": "keyword" }, - "zeek.intel.cif_firstseen": { "type": "keyword" }, - "zeek.intel.cif_lastseen": { "type": "keyword" }, - "zeek.intel.cif_source": { "type": "keyword" }, - "zeek.intel.cif_tags": { "type": "keyword" }, "zeek.intel.file_description": { "type": "keyword", "ignore_above": 1024, "fields": { "text": { "type": "text", "norms": false } } }, "zeek.intel.file_mime_type": { "type": "keyword" }, - "zeek.intel.matched": { "type": "keyword" }, - "zeek.intel.seen_indicator": { "type": "keyword" }, - "zeek.intel.seen_indicator_type": { "type": "keyword" }, "zeek.intel.seen_node": { "type": "keyword" }, "zeek.intel.seen_where": { "type": "keyword" }, - "zeek.intel.sources": { "type": "keyword" }, "zeek.ipsec.certificates": { "type": "keyword" }, "zeek.ipsec.doi": { "type": "integer" }, "zeek.ipsec.exchange_type": { "type": "integer" }, diff --git a/dashboards/templates/composable/component/zeek_ot.json b/dashboards/templates/composable/component/zeek_ot.json index 93877c03b..51a6bfea7 100644 --- a/dashboards/templates/composable/component/zeek_ot.json +++ b/dashboards/templates/composable/component/zeek_ot.json @@ -863,6 +863,7 @@ "zeek.opcua_binary.identifier": { "type": "long" }, "zeek.opcua_binary.identifier_str": { "type": "keyword" }, "zeek.opcua_binary.is_final": { "type": "keyword" }, + "zeek.opcua_binary.log_types": { "type": "integer" }, "zeek.opcua_binary.max_chunk_cnt": { "type": "long" }, "zeek.opcua_binary.max_msg_size": { "type": "long" }, "zeek.opcua_binary.msg_size": { "type": "long" }, @@ -1316,6 +1317,20 @@ "zeek.opcua_binary_variant_metadata.variant_data_array_dim": { "type": "long" }, "zeek.opcua_binary_variant_metadata.variant_data_source": { "type": "long" }, "zeek.opcua_binary_variant_metadata.variant_data_source_str": { "type": "keyword" }, + "zeek.opcua_binary_write.node_id_encoding_mask": { "type": "keyword" }, + "zeek.opcua_binary_write.node_id_namespace_idx": { "type": "long" }, + "zeek.opcua_binary_write.node_id_numeric": { "type": "long" }, + "zeek.opcua_binary_write.node_id_string": { "type": "keyword" }, + "zeek.opcua_binary_write.node_id_guid": { "type": "keyword" }, + "zeek.opcua_binary_write.node_id_opaque": { "type": "keyword" }, + "zeek.opcua_binary_write.attribute_id": { "type": "long" }, + "zeek.opcua_binary_write.attribute_id_str": { "type": "keyword" }, + "zeek.opcua_binary_write.index_range": { "type": "keyword" }, + "zeek.opcua_binary_write.data_value_encoding_mask": { "type": "keyword" }, + "zeek.opcua_binary_write.source_timestamp": { "type": "date" }, + "zeek.opcua_binary_write.source_pico_sec": { "type": "long" }, + "zeek.opcua_binary_write.server_timestamp": { "type": "date" }, + "zeek.opcua_binary_write.server_pico_sec": { "type": "long" }, "zeek.profinet.block_version": { "type": "keyword" }, "zeek.profinet.index": { "type": "keyword" }, "zeek.profinet.operation_type": { "type": "keyword" }, diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index bada0e073..8ae8694e0 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -5,7 +5,7 @@ services: build: context: . dockerfile: Dockerfiles/opensearch.Dockerfile - image: ghcr.io/idaholab/malcolm/opensearch:24.10.1 + image: ghcr.io/idaholab/malcolm/opensearch:24.11.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -83,7 +83,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards-helper.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.10.1 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -137,7 +137,7 @@ services: build: context: . dockerfile: Dockerfiles/dashboards.Dockerfile - image: ghcr.io/idaholab/malcolm/dashboards:24.10.1 + image: ghcr.io/idaholab/malcolm/dashboards:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -184,7 +184,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: ghcr.io/idaholab/malcolm/logstash-oss:24.10.1 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -290,7 +290,7 @@ services: target: /usr/share/logstash/malcolm-ruby read_only: true healthcheck: - test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9600"] + test: ["CMD", "/usr/local/bin/container_health.sh"] interval: 30s timeout: 15s retries: 3 @@ -299,7 +299,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.10.1 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -376,7 +376,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.10.1 + image: ghcr.io/idaholab/malcolm/arkime:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -462,7 +462,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: ghcr.io/idaholab/malcolm/arkime:24.10.1 + image: ghcr.io/idaholab/malcolm/arkime:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -547,7 +547,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.10.1 + image: ghcr.io/idaholab/malcolm/zeek:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -620,7 +620,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: ghcr.io/idaholab/malcolm/zeek:24.10.1 + image: ghcr.io/idaholab/malcolm/zeek:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -685,7 +685,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.10.1 + image: ghcr.io/idaholab/malcolm/suricata:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -745,7 +745,7 @@ services: build: context: . dockerfile: Dockerfiles/suricata.Dockerfile - image: ghcr.io/idaholab/malcolm/suricata:24.10.1 + image: ghcr.io/idaholab/malcolm/suricata:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -803,7 +803,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/file-monitor:24.10.1 + image: ghcr.io/idaholab/malcolm/file-monitor:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -859,7 +859,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-capture:24.10.1 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -906,7 +906,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.10.1 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -961,7 +961,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: ghcr.io/idaholab/malcolm/file-upload:24.10.1 + image: ghcr.io/idaholab/malcolm/file-upload:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1003,7 +1003,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.10.1 + image: ghcr.io/idaholab/malcolm/htadmin:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1056,7 +1056,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.10.1 + image: ghcr.io/idaholab/malcolm/freq:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1094,7 +1094,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.10.1 + image: ghcr.io/idaholab/malcolm/netbox:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1161,7 +1161,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.10.1 + image: ghcr.io/idaholab/malcolm/postgresql:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1205,7 +1205,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.10.1 + image: ghcr.io/idaholab/malcolm/redis:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1253,7 +1253,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.10.1 + image: ghcr.io/idaholab/malcolm/redis:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1296,7 +1296,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.10.1 + image: ghcr.io/idaholab/malcolm/api:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1346,7 +1346,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.10.1 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: diff --git a/docker-compose.yml b/docker-compose.yml index 8bfaae622..6128cc55d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: opensearch: - image: ghcr.io/idaholab/malcolm/opensearch:24.10.1 + image: ghcr.io/idaholab/malcolm/opensearch:24.11.0 # Technically the "hedgehog" profile doesn't have OpenSearch, but in that case # OPENSEARCH_PRIMARY will be set to remote, which means the container will # start but not actually run OpenSearch. It's included in both profiles to @@ -77,7 +77,7 @@ services: retries: 3 start_period: 180s dashboards-helper: - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.10.1 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -128,7 +128,7 @@ services: retries: 3 start_period: 30s dashboards: - image: ghcr.io/idaholab/malcolm/dashboards:24.10.1 + image: ghcr.io/idaholab/malcolm/dashboards:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -172,7 +172,7 @@ services: retries: 3 start_period: 210s logstash: - image: ghcr.io/idaholab/malcolm/logstash-oss:24.10.1 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -254,13 +254,13 @@ services: target: /certs/server.key read_only: true healthcheck: - test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9600"] + test: ["CMD", "/usr/local/bin/container_health.sh"] interval: 30s timeout: 15s retries: 3 start_period: 600s filebeat: - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.10.1 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -334,7 +334,7 @@ services: retries: 3 start_period: 60s arkime: - image: ghcr.io/idaholab/malcolm/arkime:24.10.1 + image: ghcr.io/idaholab/malcolm/arkime:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -399,7 +399,7 @@ services: retries: 3 start_period: 210s arkime-live: - image: ghcr.io/idaholab/malcolm/arkime:24.10.1 + image: ghcr.io/idaholab/malcolm/arkime:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -463,7 +463,7 @@ services: source: ./pcap target: /data/pcap zeek: - image: ghcr.io/idaholab/malcolm/zeek:24.10.1 + image: ghcr.io/idaholab/malcolm/zeek:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -527,7 +527,7 @@ services: retries: 3 start_period: 60s zeek-live: - image: ghcr.io/idaholab/malcolm/zeek:24.10.1 + image: ghcr.io/idaholab/malcolm/zeek:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -583,7 +583,7 @@ services: target: /opt/zeek/share/zeek/site/custom read_only: true suricata: - image: ghcr.io/idaholab/malcolm/suricata:24.10.1 + image: ghcr.io/idaholab/malcolm/suricata:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -640,7 +640,7 @@ services: retries: 3 start_period: 120s suricata-live: - image: ghcr.io/idaholab/malcolm/suricata:24.10.1 + image: ghcr.io/idaholab/malcolm/suricata:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -695,7 +695,7 @@ services: target: /opt/suricata/include-configs read_only: true file-monitor: - image: ghcr.io/idaholab/malcolm/file-monitor:24.10.1 + image: ghcr.io/idaholab/malcolm/file-monitor:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -748,7 +748,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: ghcr.io/idaholab/malcolm/pcap-capture:24.10.1 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -792,7 +792,7 @@ services: source: ./pcap/upload target: /pcap pcap-monitor: - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.10.1 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.11.0 profiles: ["malcolm", "hedgehog"] userns_mode: keep-id logging: @@ -844,7 +844,7 @@ services: retries: 3 start_period: 90s upload: - image: ghcr.io/idaholab/malcolm/file-upload:24.10.1 + image: ghcr.io/idaholab/malcolm/file-upload:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -886,7 +886,7 @@ services: retries: 3 start_period: 60s htadmin: - image: ghcr.io/idaholab/malcolm/htadmin:24.10.1 + image: ghcr.io/idaholab/malcolm/htadmin:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -936,7 +936,7 @@ services: retries: 3 start_period: 60s freq: - image: ghcr.io/idaholab/malcolm/freq:24.10.1 + image: ghcr.io/idaholab/malcolm/freq:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -971,7 +971,7 @@ services: retries: 3 start_period: 60s netbox: - image: ghcr.io/idaholab/malcolm/netbox:24.10.1 + image: ghcr.io/idaholab/malcolm/netbox:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1035,7 +1035,7 @@ services: retries: 3 start_period: 120s netbox-postgres: - image: ghcr.io/idaholab/malcolm/postgresql:24.10.1 + image: ghcr.io/idaholab/malcolm/postgresql:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1076,7 +1076,7 @@ services: retries: 3 start_period: 45s netbox-redis: - image: ghcr.io/idaholab/malcolm/redis:24.10.1 + image: ghcr.io/idaholab/malcolm/redis:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1121,7 +1121,7 @@ services: retries: 3 start_period: 45s netbox-redis-cache: - image: ghcr.io/idaholab/malcolm/redis:24.10.1 + image: ghcr.io/idaholab/malcolm/redis:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1161,7 +1161,7 @@ services: retries: 3 start_period: 45s api: - image: ghcr.io/idaholab/malcolm/api:24.10.1 + image: ghcr.io/idaholab/malcolm/api:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: @@ -1205,7 +1205,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.10.1 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.11.0 profiles: ["malcolm"] userns_mode: keep-id logging: diff --git a/docs/README.md b/docs/README.md index a7004736c..f8be3f476 100644 --- a/docs/README.md +++ b/docs/README.md @@ -106,6 +106,7 @@ Malcolm can also easily be deployed locally on an ordinary consumer workstation - [Zeek Intelligence Framework](zeek-intel.md#ZeekIntel) + [STIX™ and TAXII™](zeek-intel.md#ZeekIntelSTIX) + [MISP](zeek-intel.md#ZeekIntelMISP) + + [Mandiant](zeek-intel.md#ZeekIntelMandiant) - ["Best Guess" Fingerprinting for ICS Protocols](ics-best-guess.md#ICSBestGuess) - [Asset Interaction Analysis](asset-interaction-analysis.md#AssetInteractionAnalysis) + [Enriching network traffic metadata via NetBox lookups](asset-interaction-analysis.md#NetBoxEnrichment) diff --git a/docs/api-dashboard-export.md b/docs/api-dashboard-export.md new file mode 100644 index 000000000..6a4450c25 --- /dev/null +++ b/docs/api-dashboard-export.md @@ -0,0 +1,35 @@ +# Dashboard Export + +`GET` or `POST` - /mapi/dashboard-export/`` + +Uses the [OpenSearch Dashboards](https://opensearch.org/docs/latest/dashboards/) or [Elastic Kibana](https://www.elastic.co/guide/en/kibana/current/dashboard-api-export.html) API to export the JSON document representing a dashboard (identified by `dashid`). If the query parameter `replace` is not set to `false`, this API will also perform some modifications on the dashboard as described in the [**Adding new visualizations and dashboards**](contributing-dashboards.md#DashboardsNewViz) section of the [contributor guide](contributing-guide.md#Contributing). + +Parameters: + +* `dashid` (URL parameter) - the [ID of the dashboard]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/dashboards/dashboards/) to be exported (e.g., `0ad3d7c2-3441-485e-9dfe-dbb22e84e576`) +* `replace` (query parameter) - whether or not to perform the index pattern name replacements as described above (default: `true`) + +Example (response truncated for brevity's sake): + +``` +/mapi/dashboard-export/0ad3d7c2-3441-485e-9dfe-dbb22e84e576 +``` + +```json +{ + + "version": "1.3.1", + "objects": [ + { + "id": "0ad3d7c2-3441-485e-9dfe-dbb22e84e576", + "type": "dashboard", + "namespaces": [ + "default" + ], + "updated_at": "2024-04-29T15:49:16.000Z", + "version": "WzEzNjIsMV0=", + "attributes": { + "title": "Overview" +… +} +``` \ No newline at end of file diff --git a/docs/api-ingest-stats.md b/docs/api-ingest-stats.md new file mode 100644 index 000000000..14c746414 --- /dev/null +++ b/docs/api-ingest-stats.md @@ -0,0 +1,17 @@ +# Document Ingest Statistics + +`GET` - /mapi/ingest-stats + +Executes an OpenSearch [bucket aggregation](https://opensearch.org/docs/latest/opensearch/bucket-agg/) query for the `host.name` field and its maximum (i.e., most regent) `event.ingested` UTC time value for all of Malcolm's indexed network traffic metadata. + +This can be used to know the most recent time a log was indexed for each network sensor. + +Example output: + +``` +{ + "malcolm": "2024-11-04T14:58:57+00:00", + "sensor_a": "2024-11-04T14:57:41+00:00", + "sensor_b": "2024-11-04T14:58:59+00:00" +} +``` \ No newline at end of file diff --git a/docs/api-version.md b/docs/api-version.md index 2ebe0e921..c13c65c03 100644 --- a/docs/api-version.md +++ b/docs/api-version.md @@ -49,6 +49,6 @@ Returns version information about Malcolm and version/[health](https://opensearc } }, "sha": "dad18b1", - "version": "24.10.1" + "version": "24.11.0" } ``` diff --git a/docs/api.md b/docs/api.md index 16c72ee8c..cf6bb475a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1,8 +1,10 @@ # API -* [Field Aggregations](api-aggregations.md) +* [Dashboard Export](api-dashboard-export.md) +* [Document Ingest Statistics](api-ingest-stats.md) * [Document Lookup](api-document-lookup.md) * [Event Logging](api-event-logging.md) +* [Field Aggregations](api-aggregations.md) * [Fields](api-fields.md) * [Indices](api-indices.md) * [Ping](api-ping.md) diff --git a/docs/capabilities-and-limitations.md b/docs/capabilities-and-limitations.md index 7b13a9c29..9edb33f35 100644 --- a/docs/capabilities-and-limitations.md +++ b/docs/capabilities-and-limitations.md @@ -45,7 +45,7 @@ In short, Malcolm provides an easily deployable traffic analysis tool suite for - Limitation: Anomaly detection and machine learning algorithms rely on enough data (for network data, this generally means at least several weeks' worth or more) to be able to build a baseline of what is normal before they can accurately flag anomalies, and each network is different. Anomaly detection and ML are typically not useful for limited deployments without the available traffic to build that baseline. - Limitation: While Malcolm provides some powerful tools in the anomaly detection and ML realm, as of yet they have not been built out to provide the value that they will probably one day realize. * Threat ingestion - - Malcolm can ingest threat indicators in the form of static MISP- or STIX-formatted files. It can also subscribe to and periodically update threat indicators from [MISP](zeek-intel.md#ZeekIntelMISP) and [TAXII](zeek-intel.md#ZeekIntelSTIX) feeds. These indicators are converted into a format that is read by Zeek, and matches in network traffic are [surfaced through the Zeek intelligence framework](zeek-intel.md#ZeekIntel) for logging. + - Malcolm can ingest threat indicators in the form of static MISP- or STIX-formatted files. It can also subscribe to and periodically update threat indicators from [MISP](zeek-intel.md#ZeekIntelMISP), [TAXII](zeek-intel.md#ZeekIntelSTIX), and [Mandiant](zeek-intel.md#ZeekIntelMandiant) feeds. These indicators are converted into a format that is read by Zeek, and matches in network traffic are [surfaced through the Zeek intelligence framework](zeek-intel.md#ZeekIntel) for logging. - Limitation: Some formats for threat indicators allow for complex definitions and logic. For STIX/TAXII, only indicators of cyber-observable objects matched with the equals (=) comparison operator against a single value can be expressed as Zeek intelligence items. Similarly, only a subset of MISP attribute types can be expressed with the Zeek intelligence indicator types. While this is generally sufficient to cover most indicators interest, more complex indicators are silently ignored. * Network Modeling - Malcolm provides an instance of [NetBox](https://netboxlabs.com/oss/netbox/), an open-source "solution for modeling and documenting modern networks" which is used to model instrumented networks and enrich passively-observed network traffic from that model, a technique Malcolm calls ["Asset Interaction Analysis"](asset-interaction-analysis.md#AssetInteractionAnalysis). Users can pivot between the network visualization tools (the Asset Interaction Analysis and Zeek Known Summary dashboards in OpenSearch Dashboards, and the Arkime Sessions interface) and the NetBox UI to investigate and examine network assets. diff --git a/docs/contributing-dashboards.md b/docs/contributing-dashboards.md index 313e89fee..f37cb0499 100644 --- a/docs/contributing-dashboards.md +++ b/docs/contributing-dashboards.md @@ -4,36 +4,62 @@ ## Adding new visualizations and dashboards -Visualizations and dashboards can be [easily created](dashboards.md#BuildDashboard) in OpenSearch Dashboards using its drag-and-drop WYSIWIG tools. Assuming users have created a new dashboard to package with Malcolm, the dashboard and its visualization components can be exported using the following steps: +Visualizations and dashboards can be [easily created](dashboards.md#BuildDashboard) in OpenSearch Dashboards using its drag-and-drop WYSIWIG tools. Assuming users have created a new dashboard to package with Malcolm, the dashboard and its visualization components can be exported either of two ways. + +The easier (and preferred) method is to use the [dashboard export API](api-dashboard-export.md), as it handles the replacers (described below in the more complicated method) automatically.: 1. Identify the ID of the dashboard (found in the URL: e.g., for `/dashboards/app/dashboards#/view/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` the ID would be `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`) -1. Export the dashboard with that ID and save it in the `./dashboards./dashboards/` directory with the following command: - ``` - export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \ - docker compose exec dashboards curl -XGET \ - "http://localhost:5601/dashboards/api/opensearch-dashboards/dashboards/export?dashboard=$DASHID" > \ - ./dashboards/dashboards/$DASHID.json - ``` -1. It is preferrable for Malcolm to dynamically create the `arkime_sessions3-*` index template rather than including it in imported dashboards, so edit the `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` that was generated, carefully locating and removing the section with the `id` of `arkime_sessions3-*` and the `type` of `index-pattern` (including the comma preceding it): - ``` - , - { - "id": "arkime_sessions3-*", - "type": "index-pattern", - "namespaces": [ - "default" - ], - "updated_at": "2021-12-13T18:21:42.973Z", - "version": "Wzk3MSwxXQ==", - … - "references": [], - "migrationVersion": { - "index-pattern": "7.6.0" - } - } - ``` -1. In your text editor, perform a global-search and replace, replacing the string `arkime_sessions3-*` with `MALCOLM_NETWORK_INDEX_PATTERN_REPLACER` and `malcolm_beats_*` with `MALCOLM_OTHER_INDEX_PATTERN_REPLACER`. These replacers are used to [allow customizing indexes for logs written to OpenSearch or Elasticsearch](https://github.com/idaholab/Malcolm/issues/313). -1. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up. + +2. Using a web browser, enter the URL **https://localhost/mapi/dashboard-export/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx**, replacing `localhost` with the IP address or hostname of your Malcolm instance and the placeholder dashboard ID with the ID you identified in the previous step. Save the raw JSON document returned as `./dashboards/dashboards/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` (using the actual ID) under your Malcolm directory. + +**OR** + +2. Using the command line, export the dashboard with that ID and save it in the `./dashboards/dashboards/` directory with the following command: + +``` +export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \ + docker compose exec api curl -sSL -XGET "http://localhost:5000/mapi/dashboard-export/$DASHID" > \ + ./dashboards/dashboards/$DASHID.json +``` + +3. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up. + + +The manual, more complicated way, consists of the following steps: + +1. Identify the ID of the dashboard (found in the URL: e.g., for `/dashboards/app/dashboards#/view/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` the ID would be `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`) + +2. Using the command line, export the dashboard with that ID and save it in the `./dashboards/dashboards/` directory with the following command: + +``` +export DASHID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx && \ + docker compose exec dashboards curl -XGET \ + "http://localhost:5601/dashboards/api/opensearch-dashboards/dashboards/export?dashboard=$DASHID" > \ + ./dashboards/dashboards/$DASHID.json +``` + +3. It is preferrable for Malcolm to dynamically create the `arkime_sessions3-*` index template rather than including it in imported dashboards, so edit the `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.json` that was generated, carefully locating and removing the section with the `id` of `arkime_sessions3-*` and the `type` of `index-pattern` (including the comma preceding it): + +``` + , + { + "id": "arkime_sessions3-*", + "type": "index-pattern", + "namespaces": [ + "default" + ], + "updated_at": "2021-12-13T18:21:42.973Z", + "version": "Wzk3MSwxXQ==", + … + "references": [], + "migrationVersion": { + "index-pattern": "7.6.0" + } + } +``` + +4. In your text editor, perform a global-search and replace, replacing the string `arkime_sessions3-*` with `MALCOLM_NETWORK_INDEX_PATTERN_REPLACER` and `malcolm_beats_*` with `MALCOLM_OTHER_INDEX_PATTERN_REPLACER`. These replacers are used to [allow customizing indexes for logs written to OpenSearch or Elasticsearch](https://github.com/idaholab/Malcolm/issues/313). +5. Include the new dashboard either by using a [bind mount](contributing-local-modifications.md#Bind) for the `./dashboards/dashboards/` directory or by [rebuilding](development.md#Build) the `dashboards-helper` image. Dashboards are imported the first time Malcolm starts up. ## OpenSearch Dashboards plugins diff --git a/docs/contributing-github-runners.md b/docs/contributing-github-runners.md index 80a25eea3..f373bcbb2 100644 --- a/docs/contributing-github-runners.md +++ b/docs/contributing-github-runners.md @@ -104,7 +104,7 @@ Each container build workflow actually runs two paths in parallel: one for build ## Convenience scripts for development -As mentioned earlier, Malcolm images built using the instructions in this document are are named according to the pattern `ghcr.io/username/malcolm/image:branch`. However, note that the `image:` values found in [`docker-compose.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/docker-compose.yml) (and in the [Kubernetes](kubernetes.md#Kubernetes) [manifests]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/)) look like `ghcr.io/idaholab/malcolm/opensearch:24.10.1`, using the OpenSearch container as an example. To run a local instance of Malcolm using these images instead of the official `ghcr.io/idaholab` ones, users will need to edit their `docker-compose.yml` file(s) and replace the `image:` tags according to this new pattern, or use the bash helper script [`./scripts/github_image_helper.sh`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/github_image_helper.sh) to pull the repository images and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.10.1`). +As mentioned earlier, Malcolm images built using the instructions in this document are are named according to the pattern `ghcr.io/username/malcolm/image:branch`. However, note that the `image:` values found in [`docker-compose.yml`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/docker-compose.yml) (and in the [Kubernetes](kubernetes.md#Kubernetes) [manifests]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/kubernetes/)) look like `ghcr.io/idaholab/malcolm/opensearch:24.11.0`, using the OpenSearch container as an example. To run a local instance of Malcolm using these images instead of the official `ghcr.io/idaholab` ones, users will need to edit their `docker-compose.yml` file(s) and replace the `image:` tags according to this new pattern, or use the bash helper script [`./scripts/github_image_helper.sh`]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/scripts/github_image_helper.sh) to pull the repository images and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.11.0`). Before explaining that script, a discussion of the workflow files for the [Hedgehog Linux](live-analysis.md#Hedgehog) ([hedgehog-iso-build-docker-wrap-push-ghcr.yml ]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml)) and [Malcolm](malcolm-iso.md#ISO) ([malcolm-iso-build-docker-wrap-push-ghcr.yml @@ -141,9 +141,9 @@ These menu options are described below: 3. GithubTriggerPackagesBuild * This option will trigger a [repository dispatch](https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#repository_dispatch) via the [GitHub API](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#create-a-workflow-dispatch-event) using `curl`. In order for this operation to work, an environment variable named `GITHUB_OAUTH_TOKEN` must be defined containing a [personal access token](https://docs.github.com/en/rest/authentication/authenticating-to-the-rest-api?apiVersion=2022-11-28#basic-authentication) created for your GitHub user account with the "Actions (write)" repository permission. 4. PullAndTagGithubWorkflowImages - * This option will pull latest Malcolm container images (excluding the installer ISO wrapper container images) from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.10.1`) so that they may be run without modifying the local `docker-compose.yml` file. This is probably the option users will select most often. Note that this is different from the action performed in steps 1 and 0 above: this pulls the images directly from the container registry, it does **not** extract them from the Malcolm installer ISO wrapper container image. + * This option will pull latest Malcolm container images (excluding the installer ISO wrapper container images) from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.11.0`) so that they may be run without modifying the local `docker-compose.yml` file. This is probably the option users will select most often. Note that this is different from the action performed in steps 1 and 0 above: this pulls the images directly from the container registry, it does **not** extract them from the Malcolm installer ISO wrapper container image. 5. PullAndTagGithubWorkflowISOImages - * This option will pull latest Malcolm installer ISO wrapper container images from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.10.1`). + * This option will pull latest Malcolm installer ISO wrapper container images from ghcr.io for the user's Malcolm fork, and re-tag them with `ghcr.io/idaholab` and the current Malcolm version (e.g., `24.11.0`). The script can also be run non-interactively by specifying the option number on the command line (e.g., `./scripts/github_image_helper.sh 4`). @@ -182,7 +182,7 @@ $ ./scripts/github_image_helper.sh 5 PullAndTagGithubWorkflowISOImages Operation:4 PullAndTagGithubWorkflowImages -Pulling images from ghcr.io/romeogdetlevjr (main) and tagging as 24.10.1... +Pulling images from ghcr.io/romeogdetlevjr (main) and tagging as 24.11.0... main: Pulling from romeogdetlevjr/malcolm/api Digest: sha256:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Status: Image is up to date for ghcr.io/romeogdetlevjr/malcolm/api:main @@ -196,46 +196,46 @@ ghcr.io/romeogdetlevjr/malcolm/zeek:main xxxxxxxxxxxx: Pull complete ``` -Verify that the images were pulled. Note that users will see two tags for each image: one tagged with the username and branch (e.g., `ghcr.io/romeogdetlevjr/malcolm/api:main`) and another tagged with `ghcr.io/idaholab` and the Malcolm version (e.g., `ghcr.io/idaholab/malcolm/api:24.10.1`). +Verify that the images were pulled. Note that users will see two tags for each image: one tagged with the username and branch (e.g., `ghcr.io/romeogdetlevjr/malcolm/api:main`) and another tagged with `ghcr.io/idaholab` and the Malcolm version (e.g., `ghcr.io/idaholab/malcolm/api:24.11.0`). ```bash $ docker images | grep romeogdetlevjr/malcolm -ghcr.io/idaholab/malcolm/zeek 24.10.1 xxxxxxxxxxxx 10 minutes ago 1.39GB +ghcr.io/idaholab/malcolm/zeek 24.11.0 xxxxxxxxxxxx 10 minutes ago 1.39GB ghcr.io/romeogdetlevjr/malcolm/zeek main xxxxxxxxxxxx 10 minutes ago 1.39GB -ghcr.io/idaholab/malcolm/dashboards 24.10.1 xxxxxxxxxxxx 13 minutes ago 1.55GB +ghcr.io/idaholab/malcolm/dashboards 24.11.0 xxxxxxxxxxxx 13 minutes ago 1.55GB ghcr.io/romeogdetlevjr/malcolm/dashboards main xxxxxxxxxxxx 13 minutes ago 1.55GB -ghcr.io/idaholab/malcolm/suricata 24.10.1 xxxxxxxxxxxx 14 minutes ago 339MB +ghcr.io/idaholab/malcolm/suricata 24.11.0 xxxxxxxxxxxx 14 minutes ago 339MB ghcr.io/romeogdetlevjr/malcolm/suricata main xxxxxxxxxxxx 14 minutes ago 339MB -ghcr.io/idaholab/malcolm/file-monitor 24.10.1 xxxxxxxxxxxx 15 minutes ago 712MB +ghcr.io/idaholab/malcolm/file-monitor 24.11.0 xxxxxxxxxxxx 15 minutes ago 712MB ghcr.io/romeogdetlevjr/malcolm/file-monitor main xxxxxxxxxxxx 15 minutes ago 712MB -ghcr.io/idaholab/malcolm/redis 24.10.1 xxxxxxxxxxxx 15 minutes ago 55.4MB +ghcr.io/idaholab/malcolm/redis 24.11.0 xxxxxxxxxxxx 15 minutes ago 55.4MB ghcr.io/romeogdetlevjr/malcolm/redis main xxxxxxxxxxxx 15 minutes ago 55.4MB -ghcr.io/idaholab/malcolm/nginx-proxy 24.10.1 xxxxxxxxxxxx 16 minutes ago 160MB +ghcr.io/idaholab/malcolm/nginx-proxy 24.11.0 xxxxxxxxxxxx 16 minutes ago 160MB ghcr.io/romeogdetlevjr/malcolm/nginx-proxy main xxxxxxxxxxxx 16 minutes ago 160MB -ghcr.io/idaholab/malcolm/pcap-capture 24.10.1 xxxxxxxxxxxx 16 minutes ago 137MB +ghcr.io/idaholab/malcolm/pcap-capture 24.11.0 xxxxxxxxxxxx 16 minutes ago 137MB ghcr.io/romeogdetlevjr/malcolm/pcap-capture main xxxxxxxxxxxx 16 minutes ago 137MB -ghcr.io/idaholab/malcolm/htadmin 24.10.1 xxxxxxxxxxxx 16 minutes ago 246MB +ghcr.io/idaholab/malcolm/htadmin 24.11.0 xxxxxxxxxxxx 16 minutes ago 246MB ghcr.io/romeogdetlevjr/malcolm/htadmin main xxxxxxxxxxxx 16 minutes ago 246MB ghcr.io/romeogdetlevjr/malcolm/file-upload main xxxxxxxxxxxx 16 minutes ago 250MB -ghcr.io/idaholab/malcolm/file-upload 24.10.1 xxxxxxxxxxxx 16 minutes ago 250MB -ghcr.io/idaholab/malcolm/logstash-oss 24.10.1 xxxxxxxxxxxx 16 minutes ago 1.49GB +ghcr.io/idaholab/malcolm/file-upload 24.11.0 xxxxxxxxxxxx 16 minutes ago 250MB +ghcr.io/idaholab/malcolm/logstash-oss 24.11.0 xxxxxxxxxxxx 16 minutes ago 1.49GB ghcr.io/romeogdetlevjr/malcolm/logstash-oss main xxxxxxxxxxxx 16 minutes ago 1.49GB -ghcr.io/idaholab/malcolm/netbox 24.10.1 xxxxxxxxxxxx 17 minutes ago 1.66GB +ghcr.io/idaholab/malcolm/netbox 24.11.0 xxxxxxxxxxxx 17 minutes ago 1.66GB ghcr.io/romeogdetlevjr/malcolm/netbox main xxxxxxxxxxxx 17 minutes ago 1.66GB ghcr.io/romeogdetlevjr/malcolm/filebeat-oss main xxxxxxxxxxxx 18 minutes ago 405MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.10.1 xxxxxxxxxxxx 18 minutes ago 405MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.11.0 xxxxxxxxxxxx 18 minutes ago 405MB ghcr.io/romeogdetlevjr/malcolm/postgresql main xxxxxxxxxxxx 18 minutes ago 303MB -ghcr.io/idaholab/malcolm/postgresql 24.10.1 xxxxxxxxxxxx 18 minutes ago 303MB -ghcr.io/idaholab/malcolm/arkime 24.10.1 xxxxxxxxxxxx 18 minutes ago 802MB +ghcr.io/idaholab/malcolm/postgresql 24.11.0 xxxxxxxxxxxx 18 minutes ago 303MB +ghcr.io/idaholab/malcolm/arkime 24.11.0 xxxxxxxxxxxx 18 minutes ago 802MB ghcr.io/romeogdetlevjr/malcolm/arkime main xxxxxxxxxxxx 18 minutes ago 802MB -ghcr.io/idaholab/malcolm/opensearch 24.10.1 xxxxxxxxxxxx 18 minutes ago 1.42GB +ghcr.io/idaholab/malcolm/opensearch 24.11.0 xxxxxxxxxxxx 18 minutes ago 1.42GB ghcr.io/romeogdetlevjr/malcolm/opensearch main xxxxxxxxxxxx 18 minutes ago 1.42GB -ghcr.io/idaholab/malcolm/pcap-monitor 24.10.1 xxxxxxxxxxxx 18 minutes ago 176MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.11.0 xxxxxxxxxxxx 18 minutes ago 176MB ghcr.io/romeogdetlevjr/malcolm/pcap-monitor main xxxxxxxxxxxx 18 minutes ago 176MB -ghcr.io/idaholab/malcolm/dashboards-helper 24.10.1 xxxxxxxxxxxx 18 minutes ago 233MB +ghcr.io/idaholab/malcolm/dashboards-helper 24.11.0 xxxxxxxxxxxx 18 minutes ago 233MB ghcr.io/romeogdetlevjr/malcolm/dashboards-helpermain xxxxxxxxxxxx 18 minutes ago 233MB -ghcr.io/idaholab/malcolm/freq 24.10.1 xxxxxxxxxxxx 18 minutes ago 153MB +ghcr.io/idaholab/malcolm/freq 24.11.0 xxxxxxxxxxxx 18 minutes ago 153MB ghcr.io/romeogdetlevjr/malcolm/freq main xxxxxxxxxxxx 18 minutes ago 153MB -ghcr.io/idaholab/malcolm/api 24.10.1 xxxxxxxxxxxx 18 minutes ago 169MB +ghcr.io/idaholab/malcolm/api 24.11.0 xxxxxxxxxxxx 18 minutes ago 169MB ghcr.io/romeogdetlevjr/malcolm/api main xxxxxxxxxxxx 18 minutes ago 169MB ``` diff --git a/docs/contributing-pcap.md b/docs/contributing-pcap.md index 9d473d332..858bbd584 100644 --- a/docs/contributing-pcap.md +++ b/docs/contributing-pcap.md @@ -1,6 +1,6 @@ # PCAP processors -When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.10.1 release]({{ site.github.repository_url }}/releases/tag/v24.10.1)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: +When a PCAP is uploaded (either through Malcolm's [upload web interface](upload.md#Upload) or just copied manually into the `./pcap/upload` directory), the `pcap-monitor` container has a script that picks up those PCAP files and publishes to a [ZeroMQ](https://zeromq.org/) topic that can be subscribed to by any other process that wants to analyze that PCAP. In Malcolm (at the time of the [v24.11.0 release]({{ site.github.repository_url }}/releases/tag/v24.11.0)), there are three such ZeroMQ topics: the `zeek`, `suricata` and `arkime` containers. These actually share the [same script]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/pcap_processor.py) to run the PCAP through Zeek, Suricata, and Arkime, respectively. For an example to follow, the `zeek` container is the less complicated of the two. To integrate a new PCAP processing tool into Malcolm (named `cooltool` for this example) the process would entail: 1. Define the service as instructed in the [Adding a new service](contributing-new-image.md#NewImage) section * Note how the existing `zeek` and `arkime` services use [bind mounts](contributing-local-modifications.md#Bind) to access the local `./pcap` directory diff --git a/docs/contributing-release-prep.md b/docs/contributing-release-prep.md index 0ab70fa30..0d307d1a7 100644 --- a/docs/contributing-release-prep.md +++ b/docs/contributing-release-prep.md @@ -8,7 +8,7 @@ Malcolm tracks issues (whether they be bugs, new features, enhancements, etc.) f Romeo also ensures that all work towards this release has been pulled into the branch on his fork from which the release will be cut. If [pull requests]({{ site.github.repository_url }}/pulls) have been submitted upstream which resolve the issues assigned to this release, those pull requests should be merged into the branch at `romeogdetlevjr/Malcolm`, whether they were submitted initially against that fork or pulled in manually by Romeo as part of this release process. Pull requests are not accepted directly into the `main` branch of the official [upstream fork]({{ site.github.repository_url }}). In other words, the branch of Malcolm in Romeo's development fork should contain **everything** that is going to comprise this release of Malcolm. -There are several places in the Malcolm source code where the release version itself (e.g., `24.10.0`) needs to be present. Most of these places are in the documentation, consisting of markdown files, but others include [docker-compose.yml]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/docker-compose.yml), [docker-compose-dev.yml]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/docker-compose-dev.yml), and the [Kubernetes manifests]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/kubernetes). Most likely Romeo's first commit into his branch as he worked on this release was to bump those version strings ([like this](https://github.com/romeogdetlevjr/Malcolm/commit/cc7d0d8855b5cc4f04cd38ae22d1421c627444cc)), but he should verify now that he did so. +There are several places in the Malcolm source code where the release version itself (e.g., `24.10.1`) needs to be present. Most of these places are in the documentation, consisting of markdown files, but others include [docker-compose.yml]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/docker-compose.yml), [docker-compose-dev.yml]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/docker-compose-dev.yml), and the [Kubernetes manifests]({{ site.github.repository_url }}/tree/{{ site.github.build_revision }}/kubernetes). Most likely Romeo's first commit into his branch as he worked on this release was to bump those version strings ([like this](https://github.com/romeogdetlevjr/Malcolm/commit/cc7d0d8855b5cc4f04cd38ae22d1421c627444cc)), but he should verify now that he did so. ## 2. Build Malcolm container images using GitHub runners diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md index 1ca269878..7ce5f01fa 100644 --- a/docs/hedgehog-iso-build.md +++ b/docs/hedgehog-iso-build.md @@ -31,7 +31,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-24.10.1.iso" +Finished, created "/sensor-build/hedgehog-24.11.0.iso" … ``` diff --git a/docs/kubernetes.md b/docs/kubernetes.md index 8f8e3434b..e3da7ae54 100644 --- a/docs/kubernetes.md +++ b/docs/kubernetes.md @@ -272,28 +272,28 @@ agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | 861.34m | 14.36% | 19.55Gi | 9.29Gi | 61.28Gi | 11 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.10.1 | -file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.10.1 | -zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.10.1 | -dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.10.1 | -upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.10.1 | -filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.10.1 | -zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.10.1 | -logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.10.1 | -netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.10.1 | -suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.10.1 | -dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.10.1 | -netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.10.1 | -suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.10.1 | -freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.10.1 | -arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.10.1 | -pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.10.1 | -pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.10.1 | -netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.10.1 | -htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.10.1 | -netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.10.1 | -nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.10.1 | -opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.10.1 | +api-deployment-6f4686cf59-bn286 | Running | 10.42.2.14 | ReplicaSet | agent1 | 0.11m | 59.62Mi | api-container:0 | api:24.11.0 | +file-monitor-deployment-855646bd75-vk7st | Running | 10.42.2.16 | ReplicaSet | agent1 | 8.47m | 1.46Gi | file-monitor-container:0 | file-monitor:24.11.0 | +zeek-live-deployment-64b69d4b6f-947vr | Running | 10.42.2.17 | ReplicaSet | agent1 | 0.02m | 12.44Mi | zeek-live-container:0 | zeek:24.11.0 | +dashboards-helper-deployment-69dc54f6b6-ln4sq | Running | 10.42.2.15 | ReplicaSet | agent1 | 10.77m | 38.43Mi | dashboards-helper-container:0 | dashboards-helper:24.11.0 | +upload-deployment-586568844b-4jnk9 | Running | 10.42.2.18 | ReplicaSet | agent1 | 0.15m | 29.78Mi | upload-container:0 | file-upload:24.11.0 | +filebeat-deployment-6ff8bc444f-t7h49 | Running | 10.42.2.20 | ReplicaSet | agent1 | 2.84m | 70.71Mi | filebeat-container:0 | filebeat-oss:24.11.0 | +zeek-offline-deployment-844f4865bd-g2sdm | Running | 10.42.2.21 | ReplicaSet | agent1 | 0.17m | 41.92Mi | zeek-offline-container:0 | zeek:24.11.0 | +logstash-deployment-6fbc9fdcd5-hwx8s | Running | 10.42.2.22 | ReplicaSet | agent1 | 85.55m | 2.91Gi | logstash-container:0 | logstash-oss:24.11.0 | +netbox-deployment-cdcff4977-hbbw5 | Running | 10.42.2.23 | ReplicaSet | agent1 | 807.64m | 702.86Mi | netbox-container:0 | netbox:24.11.0 | +suricata-offline-deployment-6ccdb89478-z5696 | Running | 10.42.2.19 | ReplicaSet | agent1 | 0.22m | 34.88Mi | suricata-offline-container:0 | suricata:24.11.0 | +dashboards-deployment-69b5465db-vz88g | Running | 10.42.1.14 | ReplicaSet | agent2 | 0.94m | 100.12Mi | dashboards-container:0 | dashboards:24.11.0 | +netbox-redis-cache-deployment-5f77d47b8b-z7t2z | Running | 10.42.1.15 | ReplicaSet | agent2 | 3.57m | 7.36Mi | netbox-redis-cache-container:0 | redis:24.11.0 | +suricata-live-deployment-6494c77759-9rlnt | Running | 10.42.1.16 | ReplicaSet | agent2 | 0.02m | 9.69Mi | suricata-live-container:0 | suricata:24.11.0 | +freq-deployment-cfd84fd97-dnngf | Running | 10.42.1.17 | ReplicaSet | agent2 | 0.2m | 26.36Mi | freq-container:0 | freq:24.11.0 | +arkime-deployment-56999cdd66-s98pp | Running | 10.42.1.18 | ReplicaSet | agent2 | 4.15m | 113.07Mi | arkime-container:0 | arkime:24.11.0 | +pcap-monitor-deployment-594ff674c4-fsm7m | Running | 10.42.1.19 | ReplicaSet | agent2 | 1.24m | 48.44Mi | pcap-monitor-container:0 | pcap-monitor:24.11.0 | +pcap-capture-deployment-7c8bf6957-jzpzn | Running | 10.42.1.20 | ReplicaSet | agent2 | 0.02m | 9.64Mi | pcap-capture-container:0 | pcap-capture:24.11.0 | +netbox-postgres-deployment-5879b8dffc-kkt56 | Running | 10.42.1.21 | ReplicaSet | agent2 | 70.91m | 33.02Mi | netbox-postgres-container:0 | postgresql:24.11.0 | +htadmin-deployment-6fc46888b9-sq6ln | Running | 10.42.1.23 | ReplicaSet | agent2 | 0.14m | 30.53Mi | htadmin-container:0 | htadmin:24.11.0 | +netbox-redis-deployment-5bcd8f6c96-j5xpf | Running | 10.42.1.24 | ReplicaSet | agent2 | 1.46m | 7.34Mi | netbox-redis-container:0 | redis:24.11.0 | +nginx-proxy-deployment-69fcc4968d-f68tq | Running | 10.42.1.22 | ReplicaSet | agent2 | 0.31m | 22.63Mi | nginx-proxy-container:0 | nginx-proxy:24.11.0 | +opensearch-deployment-75498799f6-4zmwd | Running | 10.42.1.25 | ReplicaSet | agent2 | 89.8m | 11.03Gi | opensearch-container:0 | opensearch:24.11.0 | ``` The other control scripts (`stop`, `restart`, `logs`, etc.) work in a similar manner as in a Docker-based deployment. One notable difference is the `wipe` script: data on PersistentVolume storage cannot be deleted by `wipe`. It must be deleted manually on the storage media underlying the PersistentVolumes. @@ -554,28 +554,28 @@ agent1 | agent1 | 192.168.56.11 | agent1 | k3s | 6000m | agent2 | agent2 | 192.168.56.12 | agent2 | k3s | 6000m | 552.71m | 9.21% | 19.55Gi | 13.27Gi | 61.28Gi | 12 | Pod Name | State | Pod IP | Pod Kind | Worker Node | CPU Usage | Memory Usage | Container Name:Restarts | Container Image | -netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.10.1 | -netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.10.1 | -dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.10.1 | -freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.10.1 | -pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.10.1 | -nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.10.1 | -htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.10.1 | -opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.10.1 | -zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.10.1 | -dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.10.1 | -arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.10.1 | -api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.10.1 | -netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.10.1 | -pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.10.1 | -suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.10.1 | -suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.10.1 | -netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.10.1 | -zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.10.1 | -filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.10.1 | -file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.10.1 | -upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.10.1 | -logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.10.1 | +netbox-redis-cache-deployment-5f77d47b8b-jr9nt | Running | 10.42.2.6 | ReplicaSet | agent2 | 1.89m | 7.24Mi | netbox-redis-cache-container:0 | redis:24.11.0 | +netbox-redis-deployment-5bcd8f6c96-bkzmh | Running | 10.42.2.5 | ReplicaSet | agent2 | 1.62m | 7.52Mi | netbox-redis-container:0 | redis:24.11.0 | +dashboards-helper-deployment-69dc54f6b6-ks7ps | Running | 10.42.2.4 | ReplicaSet | agent2 | 12.95m | 40.75Mi | dashboards-helper-container:0 | dashboards-helper:24.11.0 | +freq-deployment-cfd84fd97-5bwp6 | Running | 10.42.2.8 | ReplicaSet | agent2 | 0.11m | 26.33Mi | freq-container:0 | freq:24.11.0 | +pcap-capture-deployment-7c8bf6957-hkvkn | Running | 10.42.2.12 | ReplicaSet | agent2 | 0.02m | 9.21Mi | pcap-capture-container:0 | pcap-capture:24.11.0 | +nginx-proxy-deployment-69fcc4968d-m57rz | Running | 10.42.2.10 | ReplicaSet | agent2 | 0.91m | 22.72Mi | nginx-proxy-container:0 | nginx-proxy:24.11.0 | +htadmin-deployment-6fc46888b9-vpt7l | Running | 10.42.2.7 | ReplicaSet | agent2 | 0.16m | 30.21Mi | htadmin-container:0 | htadmin:24.11.0 | +opensearch-deployment-75498799f6-5v92w | Running | 10.42.2.13 | ReplicaSet | agent2 | 139.2m | 10.86Gi | opensearch-container:0 | opensearch:24.11.0 | +zeek-live-deployment-64b69d4b6f-fcb6n | Running | 10.42.2.9 | ReplicaSet | agent2 | 0.02m | 109.55Mi | zeek-live-container:0 | zeek:24.11.0 | +dashboards-deployment-69b5465db-kgsqk | Running | 10.42.2.3 | ReplicaSet | agent2 | 14.98m | 108.85Mi | dashboards-container:0 | dashboards:24.11.0 | +arkime-deployment-56999cdd66-xxpw9 | Running | 10.42.2.11 | ReplicaSet | agent2 | 208.95m | 78.42Mi | arkime-container:0 | arkime:24.11.0 | +api-deployment-6f4686cf59-xt9md | Running | 10.42.1.3 | ReplicaSet | agent1 | 0.14m | 56.88Mi | api-container:0 | api:24.11.0 | +netbox-postgres-deployment-5879b8dffc-lb4qm | Running | 10.42.1.6 | ReplicaSet | agent1 | 141.2m | 48.02Mi | netbox-postgres-container:0 | postgresql:24.11.0 | +pcap-monitor-deployment-594ff674c4-fwq7g | Running | 10.42.1.12 | ReplicaSet | agent1 | 3.93m | 46.44Mi | pcap-monitor-container:0 | pcap-monitor:24.11.0 | +suricata-offline-deployment-6ccdb89478-j5fgj | Running | 10.42.1.10 | ReplicaSet | agent1 | 10.42m | 35.12Mi | suricata-offline-container:0 | suricata:24.11.0 | +suricata-live-deployment-6494c77759-rpt48 | Running | 10.42.1.8 | ReplicaSet | agent1 | 0.01m | 9.62Mi | suricata-live-container:0 | suricata:24.11.0 | +netbox-deployment-cdcff4977-7ns2q | Running | 10.42.1.7 | ReplicaSet | agent1 | 830.47m | 530.7Mi | netbox-container:0 | netbox:24.11.0 | +zeek-offline-deployment-844f4865bd-7x68b | Running | 10.42.1.9 | ReplicaSet | agent1 | 1.44m | 43.66Mi | zeek-offline-container:0 | zeek:24.11.0 | +filebeat-deployment-6ff8bc444f-pdgzj | Running | 10.42.1.11 | ReplicaSet | agent1 | 0.78m | 75.25Mi | filebeat-container:0 | filebeat-oss:24.11.0 | +file-monitor-deployment-855646bd75-nbngq | Running | 10.42.1.4 | ReplicaSet | agent1 | 1.69m | 1.46Gi | file-monitor-container:0 | file-monitor:24.11.0 | +upload-deployment-586568844b-9s7f5 | Running | 10.42.1.13 | ReplicaSet | agent1 | 0.14m | 29.62Mi | upload-container:0 | file-upload:24.11.0 | +logstash-deployment-6fbc9fdcd5-2hhx8 | Running | 10.42.1.5 | ReplicaSet | agent1 | 3236.29m | 357.36Mi | logstash-container:0 | logstash-oss:24.11.0 | ``` View container logs for the Malcolm deployment with `./scripts/logs` (if **[stern](https://github.com/stern/stern)** present in `$PATH`): diff --git a/docs/malcolm-config.md b/docs/malcolm-config.md index b598dc06e..55e2b7b96 100644 --- a/docs/malcolm-config.md +++ b/docs/malcolm-config.md @@ -127,7 +127,7 @@ Although the configuration script automates many of the following configuration - `ZEEK_DISABLE_ICS_ALL` and `ZEEK_DISABLE_ICS_…` - if set to `true`, these variables can be used to disable Zeek's protocol analyzers for Operational Technology/Industrial Control Systems (OT/ICS) protocols - `ZEEK_DISABLE_BEST_GUESS_ICS` - see ["Best Guess" Fingerprinting for ICS Protocols](ics-best-guess.md#ICSBestGuess) - `ZEEK_EXTRACTOR_MODE` – determines the file extraction behavior for file transfers detected by Zeek; see [Automatic file extraction and scanning](file-scanning.md#ZeekFileExtraction) for more details - - `ZEEK_INTEL_FEED_SINCE` - when querying a [TAXII](zeek-intel.md#ZeekIntelSTIX) or [MISP](zeek-intel.md#ZeekIntelMISP) feed, only process threat indicators created or modified since the time represented by this value; it may be either a fixed date/time (`01/01/2021`) or relative interval (`30 days ago`) + - `ZEEK_INTEL_FEED_SINCE` - when querying a [TAXII](zeek-intel.md#ZeekIntelSTIX), [MISP](zeek-intel.md#ZeekIntelMISP), or [Mandiant](zeek-intel.md#ZeekIntelMandiant) threat intelligence feed, only process threat indicators created or modified since the time represented by this value; it may be either a fixed date/time (`01/01/2021`) or relative interval (`30 days ago`) - `ZEEK_INTEL_ITEM_EXPIRATION` - specifies the value for Zeek's [`Intel::item_expiration`](https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html#id-Intel::item_expiration) timeout as used by the [Zeek Intelligence Framework](zeek-intel.md#ZeekIntel) (default `-1min`, which disables item expiration) - `ZEEK_INTEL_REFRESH_CRON_EXPRESSION` - specifies a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) indicating the refresh interval for generating the [Zeek Intelligence Framework](zeek-intel.md#ZeekIntel) files (defaults to empty, which disables automatic refresh) - `ZEEK_JA4SSH_PACKET_COUNT` - the Zeek [JA4+ plugin](https://github.com/FoxIO-LLC/ja4) calculates the JA4SSH value once for every *x* SSH packets; *x* is set here (default `200`) diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md index 5f32637b7..e3a4c336d 100644 --- a/docs/malcolm-iso.md +++ b/docs/malcolm-iso.md @@ -42,7 +42,7 @@ Building the ISO may take 30 minutes or more depending on the system. As the bui ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-24.10.1.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-24.11.0.iso" … ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index e83d34cad..7d7bf297d 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -60,25 +60,25 @@ You can then observe the images have been retrieved by running `docker images`: ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.10.1 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.10.1 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.10.1 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.10.1 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.10.1 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.10.1 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.10.1 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.10.1 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.10.1 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.10.1 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.10.1 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.10.1 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.10.1 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.10.1 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.10.1 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.10.1 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.10.1 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.10.1 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.10.1 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.11.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.11.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.11.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.11.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.11.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.11.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.11.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.11.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.11.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.11.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.11.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.11.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.11.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.11.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.11.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.11.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.11.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.11.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.11.0 xxxxxxxxxxxx 3 days ago 1GB ``` ### Import from pre-packaged tarballs diff --git a/docs/third-party-envs.md b/docs/third-party-envs.md index de581db19..be5377d02 100644 --- a/docs/third-party-envs.md +++ b/docs/third-party-envs.md @@ -43,7 +43,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro amazon-ebs: output will be in this color. ==> amazon-ebs: Prevalidating any provided VPC information - ==> amazon-ebs: Prevalidating AMI Name: malcolm-v24.10.1-x86_64-2024-10-10T15-41-32Z + ==> amazon-ebs: Prevalidating AMI Name: malcolm-v24.11.0-x86_64-2024-10-10T15-41-32Z amazon-ebs: Found Image ID: ami-xxxxxxxxxxxxxxxxx ... @@ -53,7 +53,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro ==> amazon-ebs: Adding tags to AMI (ami-xxxxxxxxxxxxxxxxx)... ==> amazon-ebs: Tagging snapshot: snap-xxxxxxxxxxxxxxxxx ==> amazon-ebs: Creating AMI tags - amazon-ebs: Adding tag: "Malcolm": "idaholab/Malcolm/v24.10.1" + amazon-ebs: Adding tag: "Malcolm": "idaholab/Malcolm/v24.11.0" amazon-ebs: Adding tag: "source_ami_name": "al2023-ami-ecs-hvm-2023.0.20241003-kernel-6.1-x86_64" ==> amazon-ebs: Creating snapshot tags ==> amazon-ebs: Terminating the source AWS instance... @@ -79,7 +79,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro "Architecture": "x86_64", "CreationDate": "2024-05-30T14:02:21.000Z", "ImageId": "ami-xxxxxxxxxxxxxxxxx", - "ImageLocation": "xxxxxxxxxxxx/malcolm-v24.10.1-arm64-2024-05-30T13-57-31Z", + "ImageLocation": "xxxxxxxxxxxx/malcolm-v24.11.0-arm64-2024-05-30T13-57-31Z", "ImageType": "machine", "Public": false, "OwnerId": "xxxxxxxxxxxx", @@ -100,14 +100,14 @@ The files referenced in this section can be found in [scripts/third-party-enviro ], "EnaSupport": true, "Hypervisor": "xen", - "Name": "malcolm-v24.10.1-arm64-2024-05-30T13-57-31Z", + "Name": "malcolm-v24.11.0-arm64-2024-05-30T13-57-31Z", "RootDeviceName": "/dev/xvda", "RootDeviceType": "ebs", "SriovNetSupport": "simple", "Tags": [ { "Key": "Malcolm", - "Value": "idaholab/Malcolm/v24.10.1" + "Value": "idaholab/Malcolm/v24.11.0" }, { "Key": "source_ami_name", diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md index d49211129..d1f6502ab 100644 --- a/docs/ubuntu-install-example.md +++ b/docs/ubuntu-install-example.md @@ -261,25 +261,25 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -ghcr.io/idaholab/malcolm/api 24.10.1 xxxxxxxxxxxx 3 days ago 158MB -ghcr.io/idaholab/malcolm/arkime 24.10.1 xxxxxxxxxxxx 3 days ago 816MB -ghcr.io/idaholab/malcolm/dashboards 24.10.1 xxxxxxxxxxxx 3 days ago 1.02GB -ghcr.io/idaholab/malcolm/dashboards-helper 24.10.1 xxxxxxxxxxxx 3 days ago 184MB -ghcr.io/idaholab/malcolm/file-monitor 24.10.1 xxxxxxxxxxxx 3 days ago 588MB -ghcr.io/idaholab/malcolm/file-upload 24.10.1 xxxxxxxxxxxx 3 days ago 259MB -ghcr.io/idaholab/malcolm/filebeat-oss 24.10.1 xxxxxxxxxxxx 3 days ago 624MB -ghcr.io/idaholab/malcolm/freq 24.10.1 xxxxxxxxxxxx 3 days ago 132MB -ghcr.io/idaholab/malcolm/htadmin 24.10.1 xxxxxxxxxxxx 3 days ago 242MB -ghcr.io/idaholab/malcolm/logstash-oss 24.10.1 xxxxxxxxxxxx 3 days ago 1.35GB -ghcr.io/idaholab/malcolm/netbox 24.10.1 xxxxxxxxxxxx 3 days ago 1.01GB -ghcr.io/idaholab/malcolm/nginx-proxy 24.10.1 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/opensearch 24.10.1 xxxxxxxxxxxx 3 days ago 1.17GB -ghcr.io/idaholab/malcolm/pcap-capture 24.10.1 xxxxxxxxxxxx 3 days ago 121MB -ghcr.io/idaholab/malcolm/pcap-monitor 24.10.1 xxxxxxxxxxxx 3 days ago 213MB -ghcr.io/idaholab/malcolm/postgresql 24.10.1 xxxxxxxxxxxx 3 days ago 268MB -ghcr.io/idaholab/malcolm/redis 24.10.1 xxxxxxxxxxxx 3 days ago 34.2MB -ghcr.io/idaholab/malcolm/suricata 24.10.1 xxxxxxxxxxxx 3 days ago 278MB -ghcr.io/idaholab/malcolm/zeek 24.10.1 xxxxxxxxxxxx 3 days ago 1GB +ghcr.io/idaholab/malcolm/api 24.11.0 xxxxxxxxxxxx 3 days ago 158MB +ghcr.io/idaholab/malcolm/arkime 24.11.0 xxxxxxxxxxxx 3 days ago 816MB +ghcr.io/idaholab/malcolm/dashboards 24.11.0 xxxxxxxxxxxx 3 days ago 1.02GB +ghcr.io/idaholab/malcolm/dashboards-helper 24.11.0 xxxxxxxxxxxx 3 days ago 184MB +ghcr.io/idaholab/malcolm/file-monitor 24.11.0 xxxxxxxxxxxx 3 days ago 588MB +ghcr.io/idaholab/malcolm/file-upload 24.11.0 xxxxxxxxxxxx 3 days ago 259MB +ghcr.io/idaholab/malcolm/filebeat-oss 24.11.0 xxxxxxxxxxxx 3 days ago 624MB +ghcr.io/idaholab/malcolm/freq 24.11.0 xxxxxxxxxxxx 3 days ago 132MB +ghcr.io/idaholab/malcolm/htadmin 24.11.0 xxxxxxxxxxxx 3 days ago 242MB +ghcr.io/idaholab/malcolm/logstash-oss 24.11.0 xxxxxxxxxxxx 3 days ago 1.35GB +ghcr.io/idaholab/malcolm/netbox 24.11.0 xxxxxxxxxxxx 3 days ago 1.01GB +ghcr.io/idaholab/malcolm/nginx-proxy 24.11.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/opensearch 24.11.0 xxxxxxxxxxxx 3 days ago 1.17GB +ghcr.io/idaholab/malcolm/pcap-capture 24.11.0 xxxxxxxxxxxx 3 days ago 121MB +ghcr.io/idaholab/malcolm/pcap-monitor 24.11.0 xxxxxxxxxxxx 3 days ago 213MB +ghcr.io/idaholab/malcolm/postgresql 24.11.0 xxxxxxxxxxxx 3 days ago 268MB +ghcr.io/idaholab/malcolm/redis 24.11.0 xxxxxxxxxxxx 3 days ago 34.2MB +ghcr.io/idaholab/malcolm/suricata 24.11.0 xxxxxxxxxxxx 3 days ago 278MB +ghcr.io/idaholab/malcolm/zeek 24.11.0 xxxxxxxxxxxx 3 days ago 1GB ``` Finally, start Malcolm. When Malcolm starts it will stream informational and debug messages to the console until it has completed initializing. diff --git a/docs/zeek-intel.md b/docs/zeek-intel.md index 7d00bb1c2..84010e202 100644 --- a/docs/zeek-intel.md +++ b/docs/zeek-intel.md @@ -3,12 +3,14 @@ * [Zeek Intelligence Framework](#ZeekIntel) - [STIX™ and TAXII™](#ZeekIntelSTIX) - [MISP](#ZeekIntelMISP) + - [Mandiant](#ZeekIntelMandiant) + - [Endorsement Disclaimer](#IntelFeedDisclaimer) To quote Zeek's [Intelligence Framework](https://docs.zeek.org/en/master/frameworks/intel.html) documentation, "The goals of Zeek’s Intelligence Framework are to consume intelligence data, make it available for matching, and provide infrastructure to improve performance and memory utilization. Data in the Intelligence Framework is an atomic piece of intelligence such as an IP address or an e-mail address. This atomic data will be packed with metadata such as a freeform source field, a freeform descriptive field, and a URL which might lead to more information about the specific item." Zeek [intelligence](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html) [indicator types](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type) include IP addresses, URLs, file names, hashes, email addresses, and more. Malcolm doesn't come bundled with intelligence files from any particular feed, but they can be easily included into a local instance. On [startup]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_setup.sh), Malcolm's `ghcr.io/idaholab/malcolm/zeek` container enumerates the subdirectories under `./zeek/intel` (which is [bind mounted](https://docs.docker.com/storage/bind-mounts/) into the container's runtime) and configures Zeek so those intelligence files will be automatically included in its local policy. Subdirectories under `./zeek/intel` that contain their own `__load__.zeek` file will be `@load`-ed as-is, while subdirectories containing "loose" intelligence files will be [loaded](https://docs.zeek.org/en/master/frameworks/intel.html#loading-intelligence) automatically with a `redef Intel::read_files` directive. -Note that Malcolm does not manage updates for these intelligence files. Users use the update mechanism suggested by the feeds' maintainers to keep intelligence files up to date, or use a [TAXII](#ZeekIntelSTIX) or [MISP](#ZeekIntelMISP) feed as described below. +Note that Malcolm does not manage updates for these intelligence files. Users use the update mechanism suggested by the feeds' maintainers to keep intelligence files up to date, or use a [TAXII](#ZeekIntelSTIX), [MISP](#ZeekIntelMISP), or [Mandiant](#ZeekIntelMandiant) feed as described below. Adding and deleting intelligence files under this directory will take effect upon [restarting Malcolm](running.md#StopAndRestart). Alternately, users can use the `ZEEK_INTEL_REFRESH_CRON_EXPRESSION` environment variable containing a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression) to specify the interval at which the intel files should be refreshed. This can also be done manually without restarting Malcolm by running the following command from the Malcolm installation directory: @@ -24,7 +26,23 @@ For a public example of Zeek intelligence files, see Critical Path Security's [r In addition to loading Zeek intelligence files on startup, Malcolm will [automatically generate]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_from_threat_feed.py) a Zeek intelligence file for all [Structured Threat Information Expression (STIX™)](https://oasis-open.github.io/cti-documentation/stix/intro.html) [v2.0](https://docs.oasis-open.org/cti/stix/v2.0/stix-v2.0-part1-stix-core.html)/[v2.1](https://docs.oasis-open.org/cti/stix/v2.1/stix-v2.1.html) JSON files found under `./zeek/intel/STIX`. -Additionally, if a special text file named `.stix_input.txt` is found in `./zeek/intel/STIX`, that file will be read and processed as a list of [TAXII™](https://oasis-open.github.io/cti-documentation/taxii/intro.html) [2.0](http://docs.oasis-open.org/cti/taxii/v2.0/cs01/taxii-v2.0-cs01.html)/[2.1](https://docs.oasis-open.org/cti/taxii/v2.1/csprd02/taxii-v2.1-csprd02.html) feeds, one per line, according to the following format (the username and password are optional): +Additionally, if a [YAML](https://yaml.org/) file named `taxii.yaml` is found in `./zeek/intel/STIX`, that file will be read and processed as a list of [TAXII™](https://oasis-open.github.io/cti-documentation/taxii/intro.html) [2.0](http://docs.oasis-open.org/cti/taxii/v2.0/cs01/taxii-v2.0-cs01.html)/[2.1](https://docs.oasis-open.org/cti/taxii/v2.1/csprd02/taxii-v2.1-csprd02.html) feeds. This file should minimally include: + +```yaml +- type: taxii + version: 2.1 + url: https://example.com/taxii/api2/ + collection: "*" +``` + +These other parameters can also optionally be provided: + +```yaml + username: guest + password: guest +``` + +Alternatively, if a text file named `.stix_input.txt` is found in `./zeek/intel/STIX`, that file will be read and processed as described above. The feeds are specified one per line, according to the following format (the username and password are optional): ``` taxii|version|discovery_url|collection_name|username|password @@ -42,14 +60,28 @@ Malcolm will attempt to query the TAXII feed(s) for `indicator` STIX objects and Note that only **indicators** of [**cyber-observable objects**](https://docs.oasis-open.org/cti/stix/v2.1/cs01/stix-v2.1-cs01.html#_mlbmudhl16lr) matched with the **equals (`=`)** [comparison operator](https://docs.oasis-open.org/cti/stix/v2.1/cs01/stix-v2.1-cs01.html#_t11hn314cr7w) against a **single value** can be expressed as Zeek intelligence items. More complex STIX indicators will be silently ignored. +Malcolm uses the [stix2](https://pypi.org/project/stix2/) and [taxii2-client](https://pypi.org/project/taxii2-client/) Python libraries to access STIX™/TAXII™ threat intelligence feeds. + ## MISP In addition to loading Zeek intelligence files on startup, Malcolm will [automatically generate]({{ site.github.repository_url }}/blob/{{ site.github.build_revision }}/shared/bin/zeek_intel_from_threat_feed.py) a Zeek intelligence file for all [Malware Information Sharing Platform (MISP)](https://www.misp-project.org/datamodels/) JSON files found under `./zeek/intel/MISP`. -Additionally, if a special text file named `.misp_input.txt` is found in `./zeek/intel/MISP`, that file will be read and processed as a list of [MISP feed](https://misp.gitbooks.io/misp-book/content/managing-feeds/#feeds) URLs, one per line, according to the following format: +Additionally, if a [YAML](https://yaml.org/) file named `misp.yaml` is found in `./zeek/intel/MISP`, that file will be read and processed as a list of [MISP feed](https://misp.gitbooks.io/misp-book/content/managing-feeds/#feeds) URLs. This file should minimally include: + +```yaml +- type: misp + url: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +These other parameters can also optionally be provided: + +```yaml + auth_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` +Alternatively, if a special file named `.misp_input.txt` is found in `./zeek/intel/MISP`, that file will be read and processed as described above. The feeds are specified one per line, according to the following format (the authentication key is optional): ``` -misp|misp_url|auth_key (optional) +misp|misp_url|auth_key ``` For example: @@ -72,4 +104,31 @@ Upon Malcolm connects to the URLs for the MISP feeds in `.misp_input.txt`, it wi * a list of [Events](https://www.misp-project.org/openapi/#tag/Events) returned for a request via the [MISP Automation API](https://www.misp-project.org/openapi/) made to a MISP platform's [`/events` endpoint](https://www.misp-project.org/openapi/#tag/Events/operation/restSearchEvents) * a list of [Attributes](https://www.misp-project.org/openapi/#tag/Attributes) returned for a request via the [MISP Automation API](https://www.misp-project.org/openapi/) made to a MISP platform's [`/attributes` endpoint](https://www.misp-project.org/openapi/#tag/Attributes/operation/restSearchAttributes) -Note that only a subset of MISP [attribute types](https://www.misp-project.org/datamodels/#attribute-categories-vs-types) can be expressed with the Zeek intelligence [indicator types](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type). MISP attributes with other types will be silently ignored. \ No newline at end of file +Note that only a subset of MISP [attribute types](https://www.misp-project.org/datamodels/#attribute-categories-vs-types) can be expressed with the Zeek intelligence [indicator types](https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type). MISP attributes with other types will be silently ignored. + +Malcolm uses the [MISP/PyMISP](https://github.com/MISP/PyMISP) Python library to access MISP threat intelligence feeds. + +## Mandiant + +If a [YAML](https://yaml.org/) file named `mandiant.yaml` is found in `./zeek/intel/Mandiant`, that file will be read and processed as parameters for the [Mandiant Threat Intelligence](https://www.mandiant.com/threats) service. This file should minimally include: + +```yaml +- type: mandiant + api_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + secret_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +``` + +These other parameters can also optionally be provided: + +```yaml + minimum_mscore: 60 + exclude_osint: False + bearer_token: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + api_base_url: https://api.intelligence.mandiant.com +``` + +Malcolm uses the [google/mandiant-ti-client](https://github.com/google/mandiant-ti-client) Python library to access Mandiant threat intelligence feeds. + +## Disclaimer + +Neither Malcolm's development team nor its funding sources endorse any commercial product or service, nor do they attest to the suitability or effectiveness of these products and services for any particular use case. Any reference to specific commercial products, processes, or services by trademark, manufacturer, or otherwise should not be interpreted as an endorsement, recommendation, or preference. \ No newline at end of file diff --git a/filebeat/scripts/filebeat-process-zeek-folder.sh b/filebeat/scripts/filebeat-process-zeek-folder.sh index 450baf0e2..9aae1798e 100755 --- a/filebeat/scripts/filebeat-process-zeek-folder.sh +++ b/filebeat/scripts/filebeat-process-zeek-folder.sh @@ -39,7 +39,7 @@ if mkdir $LOCKDIR; then # get new logs ready for processing cd "$ZEEK_LOGS_DIR" find . -path ./processed -prune -o -path ./current -prune -o -path ./upload -prune -o -path ./extract_files -prune -o -path ./live -prune -o -type f -exec file --separator '|' --mime-type "{}" \; | grep -P "(application/gzip|application/x-gzip|application/x-7z-compressed|application/x-bzip2|application/x-cpio|application/x-lzip|application/x-lzma|application/x-rar-compressed|application/x-tar|application/x-xz|application/zip|application/x-ms-evtx|application/octet-stream)" | sort -V | \ - xargs -n 1 -P $FILEBEAT_PREPARE_PROCESS_COUNT -I '{}' bash -c ' + xargs -P $FILEBEAT_PREPARE_PROCESS_COUNT -I '{}' bash -c ' # separate filename and mime type FILENAME="$( echo "{}" | awk -F"|" "{print \$1}" )" diff --git a/hedgehog-iso/build.sh b/hedgehog-iso/build.sh index a78b946be..8430380df 100755 --- a/hedgehog-iso/build.sh +++ b/hedgehog-iso/build.sh @@ -5,10 +5,10 @@ IMAGE_PUBLISHER=idaholab IMAGE_VERSION=1.0.0 IMAGE_DISTRIBUTION=bookworm -BEATS_VER="8.15.3" +BEATS_VER="8.16.0" BEATS_OSS="-oss" -ARKIME_VER="5.4.0" +ARKIME_VER="5.5.0" BUILD_ERROR_CODE=1 diff --git a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot index 2a992298c..f5a69daf0 100755 --- a/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0169-pip-installs.hook.chroot @@ -13,6 +13,7 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta dateparser \ debinterface \ dominate \ + git+https://github.com/google/mandiant-ti-client \ humanfriendly \ pymisp \ python-dotenv \ @@ -20,4 +21,4 @@ pip3 install --break-system-packages --no-compile --no-cache-dir --force-reinsta stix2 \ stream-zip \ taxii2-client \ - watchdog==5.0.3 + watchdog==6.0.0 diff --git a/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot index 8c40a1f7c..dea7d5351 100755 --- a/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0910-sensor-build.hook.chroot @@ -50,7 +50,7 @@ find "${ZEEK_DIR}/" -type f -exec file "{}" \; | grep -Pi "ELF 64-bit.*not strip # set up default zeek local policy and sensor-related directories cp -f /usr/local/etc/zeek/*.zeek /usr/local/etc/zeek/*.txt "${ZEEK_DIR}"/share/zeek/site/ -mkdir -p /opt/sensor/sensor_ctl/zeek/custom /opt/sensor/sensor_ctl/zeek/intel/STIX /opt/sensor/sensor_ctl/zeek/intel/MISP /opt/sensor/sensor_ctl/fluentbit +mkdir -p /opt/sensor/sensor_ctl/zeek/custom /opt/sensor/sensor_ctl/zeek/intel/STIX /opt/sensor/sensor_ctl/zeek/intel/MISP /opt/sensor/sensor_ctl/zeek/intel/Mandiant /opt/sensor/sensor_ctl/fluentbit touch /opt/sensor/sensor_ctl/zeek/intel/__load__.zeek touch /opt/sensor/sensor_ctl/zeek/custom/__load__.zeek [[ -f /usr/local/bin/zeek_intel_setup.sh ]] && mv /usr/local/bin/zeek_intel_setup.sh "${ZEEK_DIR}"/bin/ @@ -64,7 +64,7 @@ rm -Rf zeek* spicy* ### # get Arkime JA4+ plugin -ARKIME_VERSION="5.4.0" +ARKIME_VERSION="5.5.0" curl "${GITHUB_API_CURL_ARGS[@]}" \ -o "/opt/arkime/plugins/ja4plus.${ARCH}.so" \ "https://github.com/arkime/arkime/releases/download/v${ARKIME_VERSION}/ja4plus.${ARCH}.so" diff --git a/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot b/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot index 553c3da62..c24eadc13 100755 --- a/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot +++ b/hedgehog-iso/config/hooks/normal/0990-remove-unwanted-pkg.hook.chroot @@ -9,7 +9,7 @@ apt-get -y --purge remove \ libc6-dbg \ ninja-build \ sparse \ - $(dpkg --get-selections | grep -Pv "(^(dpkg|libbroker|libc6|libcrypt|libdbus|libffi|libfl|libgoogle-perftools|libgcc|libkrb5|libmaxminddb|libncurses|libnsl|libobjc|libomp|libpcap|libssl|libstdc|libtinfo|libtirpc|libunwind|libxml|libyaml|libz|linux-libc|python3|zeek|zlib1g)|deinstall$)" | cut -f1 | grep -P -- '-dev(:\w+)?$') || true + $(dpkg --get-selections | grep -Pv "(^(dpkg|libbroker|libc6|libcrypt|libdbus|libffi|libfl|libgoogle-perftools|libgcc|libkrb5|librdkafka|libmaxminddb|libncurses|libnsl|libobjc|libomp|libpcap|libssl|libstdc|libtinfo|libtirpc|libunwind|libxml|libyaml|libz|linux-libc|python3|zeek|zlib1g)|deinstall$)" | cut -f1 | grep -P -- '-dev(:\w+)?$') || true rm -rf /var/spool/ccache # remove unwanted packages diff --git a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/known-routers.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/known-routers.zeek new file mode 100644 index 000000000..b77d97b00 --- /dev/null +++ b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/known-routers.zeek @@ -0,0 +1,173 @@ +##! This script logs routers, devices that Zeek determines have +##! sent IPv4 packets with TTL 255 or IPv6 packets with HLIM 255, +##! and logs the address once per day (by default). It is modeled +##! pretty heavily after known-hosts.zeek + +@load base/utils/directions-and-hosts +@load base/frameworks/cluster + +module Known; + +export { + ## The known-routers logging stream identifier. + redef enum Log::ID += { ROUTERS_LOG }; + + ## A default logging policy hook for the stream. + global log_policy_routers: Log::PolicyHook; + + ## The record type which contains the column fields of the known-routers log. + type RouterInfo: record { + ## The timestamp at which the router was detected. + ts: time &log; + ## The originating IPv4 or IPv6 address of the detected packet with the 255 TTL/HLIM value. + orig_h: addr &log; + ## The originating MAC address of the detected packet with the 255 TTL/HLIM value. + orig_l2_addr: string &log &optional; + ## When IPv4, the TTL value. + ttl: count &log &optional; + ## When IPv6, the HLIM value. + hlim: count &log &optional; + }; + + ## Toggles between different implementations of this script. + ## When true, use a Broker data store, else use a regular Zeek set + ## with keys uniformly distributed over proxy nodes in cluster + ## operation. + const use_router_store = F &redef; + + ## The router hosts whose existence should be logged and tracked. + ## See :zeek:type:`Host` for possible choices. + option router_tracking = LOCAL_HOSTS; + + ## Holds the set of all known routers. Keys in the store are addresses + ## and their associated value will always be the "true" boolean. + global router_store: Cluster::StoreInfo; + + ## The Broker topic name to use for :zeek:see:`Known::router_store`. + const router_store_name = "zeek/known/routers" &redef; + + ## The expiry interval of new entries in :zeek:see:`Known::router_store`. + ## This also changes the interval at which routers get logged. + const router_store_expiry = 1day &redef; + + ## The timeout interval to use for operations against + ## :zeek:see:`Known::router_store`. + option router_store_timeout = 15sec; + + ## The set of all known addresses to store for preventing duplicate + ## logging of addresses. It can also be used from other scripts to + ## inspect if an address has been seen in use. + ## Maintain the list of known routers for 24 hours so that the existence + ## of each individual address is logged each day. + ## + ## In cluster operation, this set is distributed uniformly across + ## proxy nodes. + global routers: set[addr] &create_expire=1day &redef; + + ## An event that can be handled to access the :zeek:type:`Known::RouterInfo` + ## record as it is sent on to the logging framework. + global log_known_routers: event(rec: RouterInfo); +} + +event zeek_init() { + if ( ! Known::use_router_store ) + return; + Known::router_store = Cluster::create_store(Known::router_store_name); +} + +event Known::router_found(info: RouterInfo) { + if ( ! Known::use_router_store ) + return; + + when [info] ( local r = Broker::put_unique(Known::router_store$store, + info$orig_h, + T, + Known::router_store_expiry) ) { + if ( r$status == Broker::SUCCESS ) { + if ( r$result as bool ) + Log::write(Known::ROUTERS_LOG, info); + } else + Reporter::error(fmt("%s: data store put_unique failure", + Known::router_store_name)); + } + + timeout Known::router_store_timeout { + # Can't really tell if master store ended up inserting a key. + Log::write(Known::ROUTERS_LOG, info); + } +} + +event known_router_add(info: RouterInfo) { + if ( use_router_store ) + return; + + if ( info$orig_h in Known::routers ) + return; + + add Known::routers[info$orig_h]; + + @if ( ! Cluster::is_enabled() || Cluster::local_node_type() == Cluster::PROXY ) + Log::write(Known::ROUTERS_LOG, info); + @endif +} + +event Cluster::node_up(name: string, id: string) { + if ( use_router_store ) + return; + + if ( Cluster::local_node_type() != Cluster::WORKER ) + return; + + # Drop local suppression cache on workers to force HRW key repartitioning. + clear_table(Known::routers); +} + +event Cluster::node_down(name: string, id: string) { + if ( use_router_store ) + return; + + if ( Cluster::local_node_type() != Cluster::WORKER ) + return; + + # Drop local suppression cache on workers to force HRW key repartitioning. + clear_table(Known::routers); +} + +event Known::router_found(info: RouterInfo) { + if ( use_router_store ) + return; + + if ( info$orig_h in Known::routers ) + return; + + Cluster::publish_hrw(Cluster::proxy_pool, info$orig_h, known_router_add, info); + event known_router_add(info); +} + +event zeek_init() &priority=5 { + Log::create_stream(Known::ROUTERS_LOG, [$columns=RouterInfo, + $ev=log_known_routers, + $path="known_routers", + $policy=log_policy_routers]); +} + +event new_connection(c: connection) &priority=5 { + local p: raw_pkt_hdr = get_current_packet_header(); + + if ( ( ( p?$ip && ( p$ip$ttl == 255 ) ) || ( p?$ip6 && ( p$ip6$hlim == 255 ) ) ) && ( addr_matches_host(c$id$orig_h, router_tracking) ) ) { + + local ttl : count = 0; + local hlim : count = 0; + local mac : string = ""; + if ( p?$ip ) ttl = p$ip$ttl; + else if ( p?$ip6 ) hlim = p$ip6$hlim; + if ( c?$orig && c$orig?$l2_addr ) mac = c$orig$l2_addr; + + event Known::router_found([$ts=network_time(), + $orig_h=c$id$orig_h, + $ttl=ttl, + $hlim=hlim, + $orig_l2_addr=mac]); + + } +} \ No newline at end of file diff --git a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek index e456d7d15..4f697c2c9 100644 --- a/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek +++ b/hedgehog-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek @@ -12,6 +12,7 @@ global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == true_reg global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == true_regex) ? T : F; global disable_track_all_assets = (getenv("ZEEK_DISABLE_TRACK_ALL_ASSETS") == true_regex) ? T : F; global disable_best_guess_ics = (getenv("ZEEK_DISABLE_BEST_GUESS_ICS") == true_regex) ? T : F; +global disable_detect_routers = (getenv("ZEEK_DISABLE_DETECT_ROUTERS") == true_regex) ? T : F; global synchrophasor_detailed = (getenv("ZEEK_SYNCHROPHASOR_DETAILED") == true_regex) ? T : F; global synchrophasor_ports_str = getenv("ZEEK_SYNCHROPHASOR_PORTS"); global genisys_ports_str = getenv("ZEEK_GENISYS_PORTS"); @@ -44,6 +45,10 @@ global disable_ics_profinet_io_cm = (getenv("ZEEK_DISABLE_ICS_PROFINET_IO_CM") = global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == true_regex) ? T : F; global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == true_regex) ? T : F; +global zeek_kafka_enabled = (getenv("ZEEK_KAFKA_ENABLED") == true_regex) ? T : F; +global zeek_kafka_brokers = getenv("ZEEK_KAFKA_BROKERS"); +global zeek_kafka_topic = getenv("ZEEK_KAFKA_TOPIC"); + redef Broker::default_listen_address = "127.0.0.1"; redef ignore_checksums = T; @@ -105,6 +110,9 @@ global json_format = (getenv("ZEEK_JSON") == true_regex) ? T : F; @if (!disable_best_guess_ics) @load ./guess.zeek @endif +@if (!disable_detect_routers) + @load ./known-routers.zeek +@endif @load packages @if (!disable_ics_all) @@ -344,3 +352,13 @@ hook PacketAnalyzer::ECAT::log_policy_ecat_arp( filter: Log::Filter) { break; } + +@if (zeek_kafka_enabled) + @load packages/zeek-kafka + redef Kafka::send_all_active_logs = T; + redef Kafka::topic_name = zeek_kafka_topic; + redef Kafka::tag_json = T; + redef Kafka::kafka_conf = table( + ["metadata.broker.list"] = zeek_kafka_brokers +); +@endif diff --git a/hedgehog-iso/config/package-lists/build.list.chroot b/hedgehog-iso/config/package-lists/build.list.chroot index 5eadd0bfa..c1cac9e5f 100644 --- a/hedgehog-iso/config/package-lists/build.list.chroot +++ b/hedgehog-iso/config/package-lists/build.list.chroot @@ -12,6 +12,7 @@ libkrb5-dev libmagic-dev libmaxminddb-dev libnl-3-dev +librdkafka-dev libpcap-dev libsodium-dev libssl-dev diff --git a/hedgehog-iso/interface/requirements.txt b/hedgehog-iso/interface/requirements.txt index 6efc369ca..19124e269 100644 --- a/hedgehog-iso/interface/requirements.txt +++ b/hedgehog-iso/interface/requirements.txt @@ -13,4 +13,4 @@ python-dotenv==1.0.0 requests==2.32.0 six==1.16.0 urllib3==1.26.19 -Werkzeug==3.0.3 +Werkzeug==3.0.6 diff --git a/hedgehog-iso/interface/sensor_ctl/control_vars.conf b/hedgehog-iso/interface/sensor_ctl/control_vars.conf index d7c9ec680..d5249ff11 100644 --- a/hedgehog-iso/interface/sensor_ctl/control_vars.conf +++ b/hedgehog-iso/interface/sensor_ctl/control_vars.conf @@ -151,6 +151,9 @@ export ZEEK_SYNCHROPHASOR_DETAILED= export ZEEK_GENISYS_PORTS= export ZEEK_ENIP_PORTS= export ZEEK_DISABLE_BEST_GUESS_ICS=true +export ZEEK_KAFKA_ENABLED= +export ZEEK_KAFKA_BROKERS=kafka.local:9091 +export ZEEK_KAFKA_TOPIC=zeek # Suricata export SURICATA_CUSTOM_RULES_ONLY=false diff --git a/hedgehog-raspi/sensor_install.sh b/hedgehog-raspi/sensor_install.sh index f1fa87af1..c146edd73 100644 --- a/hedgehog-raspi/sensor_install.sh +++ b/hedgehog-raspi/sensor_install.sh @@ -32,9 +32,9 @@ SHARED_DIR='/opt/buildshared' WORK_DIR="$(mktemp -d -t hedgehog-XXXXXX)" SENSOR_DIR='/opt/sensor' -ARKIME_VERSION="5.4.0" +ARKIME_VERSION="5.5.0" -BEATS_VER="8.15.3" +BEATS_VER="8.16.0" BEATS_OSS="-oss" # Option to build from sources if desired diff --git a/kubernetes/03-opensearch.yml b/kubernetes/03-opensearch.yml index 815a87763..54aa07576 100644 --- a/kubernetes/03-opensearch.yml +++ b/kubernetes/03-opensearch.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: opensearch-container - image: ghcr.io/idaholab/malcolm/opensearch:24.10.1 + image: ghcr.io/idaholab/malcolm/opensearch:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: subPath: "opensearch" initContainers: - name: opensearch-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/04-dashboards.yml b/kubernetes/04-dashboards.yml index 87afa4b6e..3ca2e8e8a 100644 --- a/kubernetes/04-dashboards.yml +++ b/kubernetes/04-dashboards.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-container - image: ghcr.io/idaholab/malcolm/dashboards:24.10.1 + image: ghcr.io/idaholab/malcolm/dashboards:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/05-upload.yml b/kubernetes/05-upload.yml index 67dc3fd0e..aa10d4e96 100644 --- a/kubernetes/05-upload.yml +++ b/kubernetes/05-upload.yml @@ -34,7 +34,7 @@ spec: spec: containers: - name: upload-container - image: ghcr.io/idaholab/malcolm/file-upload:24.10.1 + image: ghcr.io/idaholab/malcolm/file-upload:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -73,7 +73,7 @@ spec: subPath: "upload" initContainers: - name: upload-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/06-pcap-monitor.yml b/kubernetes/06-pcap-monitor.yml index 03509338a..2768a3504 100644 --- a/kubernetes/06-pcap-monitor.yml +++ b/kubernetes/06-pcap-monitor.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: pcap-monitor-container - image: ghcr.io/idaholab/malcolm/pcap-monitor:24.10.1 + image: ghcr.io/idaholab/malcolm/pcap-monitor:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: pcap-monitor-zeek-volume initContainers: - name: pcap-monitor-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/07-arkime.yml b/kubernetes/07-arkime.yml index 6e4de9a78..91c652ae0 100644 --- a/kubernetes/07-arkime.yml +++ b/kubernetes/07-arkime.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: arkime-container - image: ghcr.io/idaholab/malcolm/arkime:24.10.1 + image: ghcr.io/idaholab/malcolm/arkime:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -81,7 +81,7 @@ spec: name: arkime-pcap-volume initContainers: - name: arkime-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/08-api.yml b/kubernetes/08-api.yml index 407c75e2d..46aa65638 100644 --- a/kubernetes/08-api.yml +++ b/kubernetes/08-api.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: api-container - image: ghcr.io/idaholab/malcolm/api:24.10.1 + image: ghcr.io/idaholab/malcolm/api:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/09-dashboards-helper.yml b/kubernetes/09-dashboards-helper.yml index 8b06a86fd..f3e36010a 100644 --- a/kubernetes/09-dashboards-helper.yml +++ b/kubernetes/09-dashboards-helper.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: dashboards-helper-container - image: ghcr.io/idaholab/malcolm/dashboards-helper:24.10.1 + image: ghcr.io/idaholab/malcolm/dashboards-helper:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/10-zeek.yml b/kubernetes/10-zeek.yml index b48a0700a..00b366dd0 100644 --- a/kubernetes/10-zeek.yml +++ b/kubernetes/10-zeek.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-offline-container - image: ghcr.io/idaholab/malcolm/zeek:24.10.1 + image: ghcr.io/idaholab/malcolm/zeek:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -62,7 +62,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -71,7 +71,7 @@ spec: name: process-env env: - name: PUSER_MKDIR - value: "/data/config:zeek/intel/MISP,zeek/intel/STIX;/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" + value: "/data/config:zeek/intel/Mandiant,zeek/intel/MISP,zeek/intel/STIX;/data/pcap:processed;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" volumeMounts: - name: zeek-offline-intel-volume mountPath: "/data/config" diff --git a/kubernetes/11-suricata.yml b/kubernetes/11-suricata.yml index 74df4bce9..171cba773 100644 --- a/kubernetes/11-suricata.yml +++ b/kubernetes/11-suricata.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-offline-container - image: ghcr.io/idaholab/malcolm/suricata:24.10.1 + image: ghcr.io/idaholab/malcolm/suricata:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -55,7 +55,7 @@ spec: name: suricata-offline-custom-configs-volume initContainers: - name: suricata-offline-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/12-file-monitor.yml b/kubernetes/12-file-monitor.yml index c40d56c15..36113ab80 100644 --- a/kubernetes/12-file-monitor.yml +++ b/kubernetes/12-file-monitor.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: file-monitor-container - image: ghcr.io/idaholab/malcolm/file-monitor:24.10.1 + image: ghcr.io/idaholab/malcolm/file-monitor:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: name: file-monitor-yara-rules-custom-volume initContainers: - name: file-monitor-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/13-filebeat.yml b/kubernetes/13-filebeat.yml index 4bad9aed5..fc0a8bdc1 100644 --- a/kubernetes/13-filebeat.yml +++ b/kubernetes/13-filebeat.yml @@ -33,7 +33,7 @@ spec: spec: containers: - name: filebeat-container - image: ghcr.io/idaholab/malcolm/filebeat-oss:24.10.1 + image: ghcr.io/idaholab/malcolm/filebeat-oss:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: "nginx" initContainers: - name: filebeat-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/14-logstash.yml b/kubernetes/14-logstash.yml index 88192e877..1239914e8 100644 --- a/kubernetes/14-logstash.yml +++ b/kubernetes/14-logstash.yml @@ -49,7 +49,7 @@ spec: # topologyKey: "kubernetes.io/hostname" containers: - name: logstash-container - image: ghcr.io/idaholab/malcolm/logstash-oss:24.10.1 + image: ghcr.io/idaholab/malcolm/logstash-oss:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -90,10 +90,9 @@ spec: - configMapRef: name: logstash-env livenessProbe: - httpGet: - path: / - port: 9600 - scheme: HTTP + exec: + command: + - /usr/local/bin/container_health.sh initialDelaySeconds: 600 periodSeconds: 30 timeoutSeconds: 15 @@ -115,7 +114,7 @@ spec: subPath: "logstash" initContainers: - name: logstash-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/15-netbox-redis.yml b/kubernetes/15-netbox-redis.yml index 9acc15d83..c8acd0d49 100644 --- a/kubernetes/15-netbox-redis.yml +++ b/kubernetes/15-netbox-redis.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-container - image: ghcr.io/idaholab/malcolm/redis:24.10.1 + image: ghcr.io/idaholab/malcolm/redis:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -83,7 +83,7 @@ spec: subPath: netbox/redis initContainers: - name: netbox-redis-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/16-netbox-redis-cache.yml b/kubernetes/16-netbox-redis-cache.yml index ceff58143..da5d7fc2b 100644 --- a/kubernetes/16-netbox-redis-cache.yml +++ b/kubernetes/16-netbox-redis-cache.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-redis-cache-container - image: ghcr.io/idaholab/malcolm/redis:24.10.1 + image: ghcr.io/idaholab/malcolm/redis:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/17-netbox-postgres.yml b/kubernetes/17-netbox-postgres.yml index 9beb6150c..16484c321 100644 --- a/kubernetes/17-netbox-postgres.yml +++ b/kubernetes/17-netbox-postgres.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: netbox-postgres-container - image: ghcr.io/idaholab/malcolm/postgresql:24.10.1 + image: ghcr.io/idaholab/malcolm/postgresql:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -74,7 +74,7 @@ spec: subPath: netbox/postgres initContainers: - name: netbox-postgres-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/18-netbox.yml b/kubernetes/18-netbox.yml index b2d8add31..057b6de5e 100644 --- a/kubernetes/18-netbox.yml +++ b/kubernetes/18-netbox.yml @@ -36,7 +36,7 @@ spec: spec: containers: - name: netbox-container - image: ghcr.io/idaholab/malcolm/netbox:24.10.1 + image: ghcr.io/idaholab/malcolm/netbox:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -88,7 +88,7 @@ spec: subPath: netbox/media initContainers: - name: netbox-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/19-htadmin.yml b/kubernetes/19-htadmin.yml index 7e61e21e2..c3205f0f9 100644 --- a/kubernetes/19-htadmin.yml +++ b/kubernetes/19-htadmin.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: htadmin-container - image: ghcr.io/idaholab/malcolm/htadmin:24.10.1 + image: ghcr.io/idaholab/malcolm/htadmin:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -63,7 +63,7 @@ spec: subPath: "htadmin" initContainers: - name: htadmin-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/20-pcap-capture.yml b/kubernetes/20-pcap-capture.yml index f246b88a1..4801d080f 100644 --- a/kubernetes/20-pcap-capture.yml +++ b/kubernetes/20-pcap-capture.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: pcap-capture-container - image: ghcr.io/idaholab/malcolm/pcap-capture:24.10.1 + image: ghcr.io/idaholab/malcolm/pcap-capture:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -50,7 +50,7 @@ spec: subPath: "upload" initContainers: - name: pcap-capture-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/21-zeek-live.yml b/kubernetes/21-zeek-live.yml index 3586e4044..3045705ae 100644 --- a/kubernetes/21-zeek-live.yml +++ b/kubernetes/21-zeek-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: zeek-live-container - image: ghcr.io/idaholab/malcolm/zeek:24.10.1 + image: ghcr.io/idaholab/malcolm/zeek:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -61,7 +61,7 @@ spec: subPath: "zeek/intel" initContainers: - name: zeek-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -70,7 +70,7 @@ spec: name: process-env env: - name: PUSER_MKDIR - value: "/data/config:zeek/intel/MISP,zeek/intel/STIX;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" + value: "/data/config:zeek/intel/Mandiant,zeek/intel/MISP,zeek/intel/STIX;/data/zeek-logs:current,extract_files/preserved,extract_files/quarantine,live,processed,upload" volumeMounts: - name: zeek-live-intel-volume mountPath: "/data/config" diff --git a/kubernetes/22-suricata-live.yml b/kubernetes/22-suricata-live.yml index f587c370b..f29892fdc 100644 --- a/kubernetes/22-suricata-live.yml +++ b/kubernetes/22-suricata-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: suricata-live-container - image: ghcr.io/idaholab/malcolm/suricata:24.10.1 + image: ghcr.io/idaholab/malcolm/suricata:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -56,7 +56,7 @@ spec: name: suricata-live-custom-configs-volume initContainers: - name: suricata-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/23-arkime-live.yml b/kubernetes/23-arkime-live.yml index 2b0a72754..108df126b 100644 --- a/kubernetes/23-arkime-live.yml +++ b/kubernetes/23-arkime-live.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: arkime-live-container - image: ghcr.io/idaholab/malcolm/arkime:24.10.1 + image: ghcr.io/idaholab/malcolm/arkime:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -64,7 +64,7 @@ spec: name: arkime-live-pcap-volume initContainers: - name: arkime-live-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/24-freq.yml b/kubernetes/24-freq.yml index c88134128..f0f88c387 100644 --- a/kubernetes/24-freq.yml +++ b/kubernetes/24-freq.yml @@ -30,7 +30,7 @@ spec: spec: containers: - name: freq-container - image: ghcr.io/idaholab/malcolm/freq:24.10.1 + image: ghcr.io/idaholab/malcolm/freq:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/kubernetes/98-nginx-proxy.yml b/kubernetes/98-nginx-proxy.yml index b1d97fb4c..6ec382c6d 100644 --- a/kubernetes/98-nginx-proxy.yml +++ b/kubernetes/98-nginx-proxy.yml @@ -39,7 +39,7 @@ spec: spec: containers: - name: nginx-proxy-container - image: ghcr.io/idaholab/malcolm/nginx-proxy:24.10.1 + image: ghcr.io/idaholab/malcolm/nginx-proxy:24.11.0 imagePullPolicy: Always stdin: false tty: true @@ -99,7 +99,7 @@ spec: subPath: "nginx" initContainers: - name: nginx-dirinit-container - image: ghcr.io/idaholab/malcolm/dirinit:24.10.1 + image: ghcr.io/idaholab/malcolm/dirinit:24.11.0 imagePullPolicy: Always stdin: false tty: true diff --git a/logstash/maps/zeek_intel_indicator_types.yaml b/logstash/maps/zeek_intel_indicator_types.yaml new file mode 100644 index 000000000..5b2819693 --- /dev/null +++ b/logstash/maps/zeek_intel_indicator_types.yaml @@ -0,0 +1,11 @@ +"Intel::DOMAIN": "domain-name" +"Intel::EMAIL": "email-addr" +"Intel::FILE_NAME": "file" +"Intel::FILE_HASH": "file" +"Intel::ADDR": "ipv4-addr" +"Intel::ADDR4": "ipv4-addr" +"Intel::ADDR6": "ipv6-addr" +"Intel::SOFTWARE": "software" +"Intel::URL": "url" +"Intel::USER_NAME": "user-account" +"Intel::CERT_HASH": "x509-certificate" \ No newline at end of file diff --git a/logstash/pipelines/zeek/1029_zeek_intel.conf b/logstash/pipelines/zeek/1029_zeek_intel.conf index d284f10a4..da376c60e 100644 --- a/logstash/pipelines/zeek/1029_zeek_intel.conf +++ b/logstash/pipelines/zeek/1029_zeek_intel.conf @@ -40,16 +40,15 @@ filter { code => "event.set('[zeek_cols]', @zeek_intel_field_names.zip(event.get('[message]')).to_h)" } } - mutate { id => "mutate_split_zeek_intel_commas" - split => { "[zeek_cols][sources]" => "," - "[zeek_cols][matched]" => "," } } } - # For some reason, even in JSON, I have cif_tags strings like: - # Network activity,osint:source-type=\"block-or-filter-list\" - # so whatever reason it's not already an array. Split it here. - mutate { id => "mutate_split_zeek_intel_cif_tags" - split => { "[zeek_cols][cif_tags]" => "," } } + # "sources" is handled differently because of some weirdness between creating the meta.source field in the + # intel file (which seems to be a string) and the sources field in intel.log (which is a set of string) + # so we're doing our own | pseudo-delimiter to work around it. + mutate { id => "mutate_split_zeek_intel_commas" + split => { "[zeek_cols][sources]" => "|" + "[zeek_cols][matched]" => "," + "[zeek_cols][cif_tags]" => "," } } } diff --git a/logstash/pipelines/zeek/1043_zeek_opcua_binary.conf b/logstash/pipelines/zeek/1043_zeek_opcua_binary.conf index b9431e3d2..59e70689f 100644 --- a/logstash/pipelines/zeek/1043_zeek_opcua_binary.conf +++ b/logstash/pipelines/zeek/1043_zeek_opcua_binary.conf @@ -1569,6 +1569,41 @@ filter { add_tag => [ "ics" ] } + } else if ([log_source] == "opcua_binary_write") { + ############################################################################################################################# + # opcua_binary_write.log + # write-types.zeek (https://github.com/cisagov/icsnpp-opcua-binary) + + if ("_jsonparsesuccess" not in [tags]) { + dissect { + id => "dissect_zeek_opcua_binary_write" + mapping => { + "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][drop_orig_h]} %{[zeek_cols][drop_orig_p]} %{[zeek_cols][drop_resp_h]} %{[zeek_cols][drop_resp_p]} %{[zeek_cols][is_orig]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][opcua_link_id]} %{[zeek_cols][node_id_encoding_mask]} %{[zeek_cols][node_id_namespace_idx]} %{[zeek_cols][node_id_numeric]} %{[zeek_cols][node_id_string]} %{[zeek_cols][node_id_guid]} %{[zeek_cols][node_id_opaque]} %{[zeek_cols][attribute_id]} %{[zeek_cols][attribute_id_str]} %{[zeek_cols][index_range]} %{[zeek_cols][data_value_encoding_mask]} %{[zeek_cols][req_status_code_link_id]} %{[zeek_cols][source_timestamp]} %{[zeek_cols][source_pico_sec]} %{[zeek_cols][server_timestamp]} %{[zeek_cols][server_pico_sec]} %{[zeek_cols][write_results_variant_metadata_link_id]} %{[zeek_cols][res_status_code_link_id]} %{[zeek_cols][diag_info_link_id]}" + } + } + + if ("_dissectfailure" in [tags]) { + mutate { + id => "mutate_split_zeek_opcua_binary_write" + split => { "[message]" => " " } + } + ruby { + id => "ruby_zip_zeek_opcua_binary_write" + init => "$zeek_opcua_binary_write_field_names = [ 'ts', 'uid', 'drop_orig_h', 'drop_orig_p', 'drop_resp_h', 'drop_resp_p', 'is_orig', 'orig_h', 'orig_p', 'resp_p', 'opcua_link_id', 'node_id_encoding_mask', 'node_id_namespace_idx', 'node_id_numeric', 'node_id_string', 'node_id_guid', 'node_id_opaque', 'attribute_id', 'attribute_id_str', 'index_range', 'data_value_encoding_mask', 'req_status_code_link_id', 'source_timestamp', 'source_pico_sec', 'server_timestamp', 'server_pico_sec', 'write_results_variant_metadata_link_id', 'res_status_code_link_id', 'diag_info_link_id' ]" + code => "event.set('[zeek_cols]', $zeek_opcua_binary_write_field_names.zip(event.get('[message]')).to_h)" + } + } + } + + mutate { + id => "mutate_add_fields_zeek_opcua_binary_write" + add_field => { + "[zeek_cols][proto]" => "tcp" + "[zeek_cols][service]" => "opcua-binary" + } + add_tag => [ "ics" ] + } + } else { # some other unknown zeek opcua- log file. should start with ts at least! diff --git a/logstash/pipelines/zeek/1200_zeek_mutate.conf b/logstash/pipelines/zeek/1200_zeek_mutate.conf index 434e62eae..e3f7ae622 100644 --- a/logstash/pipelines/zeek/1200_zeek_mutate.conf +++ b/logstash/pipelines/zeek/1200_zeek_mutate.conf @@ -1033,10 +1033,18 @@ filter { code => " if (sources = event.get('[zeek][intel][sources]')) and (sources.length > 0) then sourcesAdj = Array.new + indicatorRef = Array.new unless (indicatorRef = event.get('[threat][indicator][reference]')) + indicatorRef = indicatorRef.is_a?(Array) ? indicatorRef : [indicatorRef] sources.each do |val| - sourcesAdj.push(val.gsub('\\x2c', ',')) + adjVal = val.gsub('\\x2c', ',') + if adjVal.match?(/^http?s:/i) + indicatorRef.push(adjVal) + else + sourcesAdj.push(adjVal) + end end - event.set('[zeek][intel][sources]', sourcesAdj) + event.set('[zeek][intel][sources]', sourcesAdj) unless (sourcesAdj.length == 0) + event.set('[threat][indicator][reference]', indicatorRef) unless (indicatorRef.length == 0) end if (tags = event.get('[zeek][intel][cif_tags]')) and (tags.length > 0) then tagsAdj = Array.new @@ -1594,6 +1602,19 @@ filter { rename => { "[zeek][opcua_binary_event_filter_attribute_operand][browse_path_element_link_id]" => "[zeek][opcua_binary_event_filter_attribute_operand_browse_paths][browse_path_element_link_id]" } rename => { "[zeek][opcua_binary_event_filter_element_operand][content_filter_filter_operand_link_id]" => "[zeek][opcua_binary][operand_source_link_id]" } rename => { "[zeek][opcua_binary_event_filter_literal_operand][content_filter_filter_operand_link_id]" => "[zeek][opcua_binary][operand_source_link_id]" } + rename => { "[zeek][opcua_binary_write][opcua_link_id]" => "[zeek][opcua_binary][opcua_link_id]" } + rename => { "[zeek][opcua_binary_write][diag_info_link_id]" => "[zeek][opcua_binary_diag_info_detail][diag_info_link_id]" } + rename => { "[zeek][opcua_binary_write][write_results_variant_metadata_link_id]" => "[zeek][opcua_binary][variant_source_link_id]" } + } + if ([zeek][opcua_binary_write][req_status_code_link_id]) { + mutate { id => "mutate_merge_zeek_opcua_binary_write_req_status_code_link_id" + merge => { "[zeek][opcua_binary_status_code_detail][status_code_link_id]" => "[zeek][opcua_binary_write][req_status_code_link_id]" } + remove_field => [ "[zeek][opcua_binary_write][req_status_code_link_id]" ] } + } + if ([zeek][opcua_binary_write][res_status_code_link_id]) { + mutate { id => "mutate_merge_zeek_opcua_binary_write_res_status_code_link_id" + merge => { "[zeek][opcua_binary_status_code_detail][status_code_link_id]" => "[zeek][opcua_binary_write][res_status_code_link_id]" } + remove_field => [ "[zeek][opcua_binary_write][res_status_code_link_id]" ] } } # count the number of contributing "log types" after we've renamed stuff diff --git a/logstash/pipelines/zeek/1300_zeek_normalize.conf b/logstash/pipelines/zeek/1300_zeek_normalize.conf index c7b1c8bc2..8deb42be8 100644 --- a/logstash/pipelines/zeek/1300_zeek_normalize.conf +++ b/logstash/pipelines/zeek/1300_zeek_normalize.conf @@ -2,70 +2,183 @@ filter { - # Zeek intel indicators depending on indicator type ################################################################# - if ([zeek][intel][seen_indicator]) and ([zeek][intel][seen_indicator_type]) { - - if ([zeek][intel][seen_indicator_type] == "Intel::ADDR") { - # ECS - ADDR-type intel indicators to related.ip - mutate { id => "mutate_merge_field_related_ip_zeek_intel_addr" - merge => { "[related][ip]" => "[zeek][intel][seen_indicator]" } } - - } else if ([zeek][intel][seen_indicator_type] == "Intel::URL") { - # ECS - URL-type intel indicators to url.original - if (![url][original]) { - mutate { id => "mutate_merge_field_related_ip_zeek_intel_url" - merge => { "[url][original]" => "[zeek][intel][seen_indicator]" } } + # most Zeek intel stuff maps to ecs-threat + # https://www.elastic.co/guide/en/ecs/current/ecs-threat.html + if ([zeek][intel]) { + + # map confidence (0-1) to Low/Medium/High/None + if ([zeek][intel][cif_confidence]) { + ruby { + id => "ruby_map_zeek_intel_confidence" + code => " + if (confNumVal = event.get('[zeek][intel][cif_confidence]').to_f) then + confStrVal = case confNumVal + when 0...3 + 'Low' + when 3...7 + 'Medium' + when 7...10 + 'High' + else + 'None' + end + event.set('[threat][indicator][confidence]', confStrVal) + end" } + } + + # normalize indicator type and indicator value for ECS + if ([zeek][intel][seen_indicator_type]) { + + # store indicator value in ECS-mapped fields + if ([zeek][intel][seen_indicator]) { + + if ([zeek][intel][seen_indicator_type] == "Intel::ADDR") { + # ECS - ADDR-type intel indicators to related.ip + mutate { id => "mutate_merge_field_related_ip_zeek_intel_addr" + merge => { "[related][ip]" => "[zeek][intel][seen_indicator]" } } + # determine if the seen indicator is ipv4 or ipv6 + cidr { + id => "cidr_detect_network_type_ipv4_source" + add_field => { "[@metadata][intel_indicator_ip_type]" => "Intel::ADDR4" } + address => [ "%{[zeek][intel][seen_indicator]}" ] + network => [ "0.0.0.0/0" ] + } + if (![@metadata][intel_indicator_ip_type]) { + mutate { id => "cidr_detect_network_type_ipv6_source" + add_field => { "[@metadata][intel_indicator_ip_type]" => "Intel::ADDR6" } } + } + mutate { id => "mutate_replace_zeek_intel_addr_indicator_type" + replace => { "[zeek][intel][seen_indicator_type]" => "%{[@metadata][intel_indicator_ip_type]}" } } + + } else if ([zeek][intel][seen_indicator_type] == "Intel::URL") { + # ECS - URL-type intel indicators to url.original + if (![url][original]) { + mutate { id => "mutate_merge_field_related_ip_zeek_intel_url" + merge => { "[url][original]" => "[zeek][intel][seen_indicator]" } } + } + + } else if ([zeek][intel][seen_indicator_type] == "Intel::DOMAIN") { + # ECS - DOMAIN-type intel indicators to related.hosts + mutate { id => "mutate_merge_field_related_ip_zeek_intel_domain" + merge => { "[related][hosts]" => "[zeek][intel][seen_indicator]" } } + + } else if ([zeek][intel][seen_indicator_type] == "Intel::USER_NAME") { + # ECS - USER_NAME-type intel indicators to related.user + mutate { id => "mutate_merge_field_related_ip_zeek_intel_user" + merge => { "[related][user]" => "[zeek][intel][seen_indicator]" } } + + } else if ([zeek][intel][seen_indicator_type] == "Intel::FILE_HASH") { + # ECS - FILE_HASH-type intel indicators to related.hash + mutate { id => "mutate_merge_field_related_ip_zeek_intel_hash" + merge => { "[related][hash]" => "[zeek][intel][seen_indicator]" } } + + } else if ([zeek][intel][seen_indicator_type] == "Intel::FILE_NAME") { + # ECS - FILE_NAME-type intel indicators to file.path + if (![file][path]) { + mutate { id => "mutate_merge_field_related_ip_zeek_intel_file_name" + merge => { "[file][path]" => "[zeek][intel][seen_indicator]" } } + } + } + + mutate { id => "mutate_suricata_rename_zeek_intel_seen_indicator" + rename => { "[zeek][intel][seen_indicator]" => "[threat][indicator][name]" } } + } # seen_indicator + + # map zeek intel indicator type from seen.indicator_type to ECS-specified values + # https://www.elastic.co/guide/en/ecs/current/ecs-threat.html#field-threat-enrichments-indicator-type + translate { + id => "translate_intel_seen_indicator_type_seen" + source => "[zeek][intel][seen_indicator_type]" + target => "[@metadata][intel_indicator_type_seen]" + dictionary_path => "/etc/zeek_intel_indicator_types.yaml" + remove_field => [ "[zeek][intel][seen_indicator_type]" ] + } + + } # seen_indicator_type - } else if ([zeek][intel][seen_indicator_type] == "Intel::DOMAIN") { - # ECS - DOMAIN-type intel indicators to related.hosts - mutate { id => "mutate_merge_field_related_ip_zeek_intel_domain" - merge => { "[related][hosts]" => "[zeek][intel][seen_indicator]" } } - - } else if ([zeek][intel][seen_indicator_type] == "Intel::USER_NAME") { - # ECS - USER_NAME-type intel indicators to related.user - mutate { id => "mutate_merge_field_related_ip_zeek_intel_user" - merge => { "[related][user]" => "[zeek][intel][seen_indicator]" } } - - } else if ([zeek][intel][seen_indicator_type] == "Intel::FILE_HASH") { - # ECS - FILE_HASH-type intel indicators to related.hash - mutate { id => "mutate_merge_field_related_ip_zeek_intel_hash" - merge => { "[related][hash]" => "[zeek][intel][seen_indicator]" } } - - } else if ([zeek][intel][seen_indicator_type] == "Intel::FILE_NAME") { - # ECS - FILE_NAME-type intel indicators to file.path - if (![file][path]) { - mutate { id => "mutate_merge_field_related_ip_zeek_intel_file_name" - merge => { "[file][path]" => "[zeek][intel][seen_indicator]" } } + if ([zeek][intel][matched]) { + # map zeek intel indicator type from seen.matched to ECS-specified values + # https://www.elastic.co/guide/en/ecs/current/ecs-threat.html#field-threat-enrichments-indicator-type + translate { + id => "translate_intel_seen_indicator_types_seen" + iterate_on => "[zeek][intel][matched]" + source => "[zeek][intel][matched]" + target => "[@metadata][intel_indicator_types_matched]" + dictionary_path => "/etc/zeek_intel_indicator_types.yaml" + remove_field => [ "[zeek][intel][matched]" ] } } - } - if ([zeek][intel][seen_where] =~ /^(DNS|HTTP|SMTP|SSL|SSH|SMB|X509)::/) and (![network][protocol]) { - # use zeel.intel.seen_where to map to service - # https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Wheres - grok { - id => "grok_zeek_intel_seen_where" - match => { "[zeek][intel][seen_where]" => [ "^%{DATA:[@metadata][intel_seen_where]}::%{GREEDYDATA:[@metadata][intel_seen_where_sub]}$" ] } + # merge indicator types found above to threat.indicator.type + if ([@metadata][intel_indicator_type_seen]) { + mutate { id => "mutate_merge_zeek_intel_indicator_type_seen" + merge => { "[threat][indicator][type]" => "[@metadata][intel_indicator_type_seen]" } } } - if ([@metadata][intel_seen_where]) { - translate { - id => "translate_zeek_intel_seen_where" - source => "[@metadata][intel_seen_where]" - target => "[network][protocol]" - dictionary => { - "DNS" => "dns" - "HTTP" => "http" - "SMTP" => "smtp" - "SSL" => "tls" - "SSH" => "ssh" - "SMB" => "smb" - "X509" => "X.509" + if ([@metadata][intel_indicator_types_matched]) { + mutate { id => "mutate_merge_zeek_intel_indicator_types_matched" + merge => { "[threat][indicator][type]" => "[@metadata][intel_indicator_types_matched]" } } + } + if ([threat][indicator][type]) { + ruby { + id => "ruby_threat_indicator_type_uniq" + path => "/usr/share/logstash/malcolm-ruby/make_unique_array.rb" + script_params => { + "field" => "[threat][indicator][type]" } - fallback => "%{[@metadata][intel_seen_where]}" } } - } + + # zeek.intel.sources -> threat.indicator.provider + if ([zeek][intel][sources]) { + mutate { id => "mutate_merge_zeek_intel_sources" + merge => { "[threat][indicator][provider]" => "[zeek][intel][sources]" } + remove_field => [ "[zeek][intel][sources]" ] } + } + # zeek.intel.cif_source -> threat.indicator.provider + if ([zeek][intel][cif_source]) { + mutate { id => "mutate_merge_zeek_intel_cif_source" + merge => { "[threat][indicator][provider]" => "[zeek][intel][cif_source]" } + remove_field => [ "[zeek][intel][cif_source]" ] } + } + # zeek.intel.cif_tags -> tags + if ([zeek][intel][cif_tags]) { + mutate { id => "mutate_merge_zeek_intel_cif_tags" + merge => { "[tags]" => "[zeek][intel][cif_tags]" } + remove_field => [ "[zeek][intel][cif_tags]" ] } + } + # zeek.intel.cif_description -> threat.indicator.description + if ([zeek][intel][cif_description]) { + mutate { id => "mutate_suricata_rename_zeek_intel_cif_description" + rename => { "[zeek][intel][cif_description]" => "[threat][indicator][description]" } } + } + + if ([zeek][intel][seen_where] =~ /^(DNS|HTTP|SMTP|SSL|SSH|SMB|X509)::/) and (![network][protocol]) { + # use zeel.intel.seen_where to map to service + # https://docs.zeek.org/en/master/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Wheres + grok { + id => "grok_zeek_intel_seen_where" + match => { "[zeek][intel][seen_where]" => [ "^%{DATA:[@metadata][intel_seen_where]}::%{GREEDYDATA:[@metadata][intel_seen_where_sub]}$" ] } + } + if ([@metadata][intel_seen_where]) { + translate { + id => "translate_zeek_intel_seen_where" + source => "[@metadata][intel_seen_where]" + target => "[network][protocol]" + dictionary => { + "DNS" => "dns" + "HTTP" => "http" + "SMTP" => "smtp" + "SSL" => "tls" + "SSH" => "ssh" + "SMB" => "smb" + "X509" => "X.509" + } + fallback => "%{[@metadata][intel_seen_where]}" + } + } + } + } # zeek intel # Protocol/service version ########################################################################################## # collect protocol version under the parent network.protocol_version array @@ -467,6 +580,9 @@ filter { } else if ([event][dataset] =~ /^opcua_binary_read/) { mutate { id => "mutate_add_field_metadata_opcua_read" add_field => { "[@metadata][opcua_action_from_dataset]" => "Read" } } + } else if ([event][dataset] =~ /^opcua_binary_write/) { + mutate { id => "mutate_add_field_metadata_opcua_write" + add_field => { "[@metadata][opcua_action_from_dataset]" => "Write" } } } if ([@metadata][opcua_action_from_dataset]) { mutate { id => "mutate_merge_zeek_opcua_action_from_dataset" diff --git a/logstash/pipelines/zeek/1400_zeek_convert.conf b/logstash/pipelines/zeek/1400_zeek_convert.conf index 7fc747619..28d2c6cb4 100644 --- a/logstash/pipelines/zeek/1400_zeek_convert.conf +++ b/logstash/pipelines/zeek/1400_zeek_convert.conf @@ -281,6 +281,31 @@ filter { } } + if ([zeek][opcua_binary_write][server_timestamp]) { + if ([zeek][opcua_binary_write][server_timestamp] == "0.000000") { + mutate { id => "mutate_remove_zeek_opcua_binary_write_server_timestamp" + remove_field => [ "[zeek][opcua_binary_write][server_timestamp]" ] } + } else { + date { + id => "date_zeek_zeek_opcua_binary_write_server_timestamp" + match => [ "[zeek][opcua_binary_write][server_timestamp]", "UNIX" ] + target => "[zeek][opcua_binary_write][server_timestamp]" + } + } + } + + if ([zeek][opcua_binary_write][source_timestamp]) { + if ([zeek][opcua_binary_write][source_timestamp] == "0.000000") { + mutate { id => "mutate_remove_zeek_opcua_binary_write_source_timestamp" + remove_field => [ "[zeek][opcua_binary_write][source_timestamp]" ] } + } else { + date { + id => "date_zeek_zeek_opcua_binary_write_source_timestamp" + match => [ "[zeek][opcua_binary_write][source_timestamp]", "UNIX" ] + target => "[zeek][opcua_binary_write][source_timestamp]" + } + } + } if ([zeek][pe][compile_ts]) { if ([zeek][pe][compile_ts] == "0.000000") { @@ -464,4 +489,31 @@ filter { } } + if ([zeek][intel][cif_firstseen]) { + if ([zeek][intel][cif_firstseen] == "0.000000") { + mutate { id => "mutate_remove_field_zeek_intel_cif_firstseen_zero" + remove_field => [ "[zeek][intel][cif_firstseen]" ] } + } else { + date { + id => "date_zeek_intel_cif_firstseen" + match => [ "[zeek][intel][cif_firstseen]", "UNIX" ] + target => "[threat][indicator][first_seen]" + remove_field => [ "[zeek][intel][cif_firstseen]" ] + } + } + } + if ([zeek][intel][cif_lastseen]) { + if ([zeek][intel][cif_lastseen] == "0.000000") { + mutate { id => "mutate_remove_field_zeek_intel_cif_lastseen_zero" + remove_field => [ "[zeek][intel][cif_lastseen]" ] } + } else { + date { + id => "date_zeek_intel_cif_lastseen" + match => [ "[zeek][intel][cif_lastseen]", "UNIX" ] + target => "[threat][indicator][last_seen]" + remove_field => [ "[zeek][intel][cif_lastseen]" ] + } + } + } + } \ No newline at end of file diff --git a/logstash/scripts/container_health.sh b/logstash/scripts/container_health.sh new file mode 100755 index 000000000..de11eb407 --- /dev/null +++ b/logstash/scripts/container_health.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +JQ_EVAL=$( + curl --fail --silent -XGET http://localhost:9600/_health_report | \ + jq '(.status == "green") and + (.indicators.pipelines.status == "green") and + (all(.indicators.pipelines.indicators[]; .status == "green"))' 2>/dev/null +) + +[[ "$JQ_EVAL" == "true" ]] && exit 0 && exit 1 \ No newline at end of file diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh index db293d456..e737d13e8 100755 --- a/malcolm-iso/build.sh +++ b/malcolm-iso/build.sh @@ -127,6 +127,7 @@ if [ -d "$WORKDIR" ]; then mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/processed/" mkdir -p "$MALCOLM_DEST_DIR/zeek-logs/upload/" mkdir -p "$MALCOLM_DEST_DIR/zeek/custom/" + mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/Mandiant/" mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/MISP/" mkdir -p "$MALCOLM_DEST_DIR/zeek/intel/STIX/" cp ./docker-compose.yml "$MALCOLM_DEST_DIR/docker-compose.yml" diff --git a/netbox/scripts/netbox_install_plugins.py b/netbox/scripts/netbox_install_plugins.py index 02524ad71..60960a77f 100755 --- a/netbox/scripts/netbox_install_plugins.py +++ b/netbox/scripts/netbox_install_plugins.py @@ -189,6 +189,17 @@ def main(): netboxVenvPy = os.path.join(os.path.join(os.path.join(args.netboxDir, 'venv'), 'bin'), 'python') manageScript = os.path.join(os.path.join(args.netboxDir, 'netbox'), 'manage.py') + # set a variable in local_settings.py for netbox-helathcheck-plugin + # see https://github.com/netbox-community/netbox-healthcheck-plugin/issues/12#issuecomment-2451665212 + netboxSettingsPyDir = os.path.join(args.netboxDir, os.path.join('netbox', 'netbox')) + if os.path.isdir(netboxSettingsPyDir): + try: + localSettingsPyContents = "import os\n\nREDIS_URL = f\"redis://{os.environ.get('REDIS_USERNAME', '')}:{os.environ.get('REDIS_PASSWORD', '')}@{os.environ.get('REDIS_HOST', 'netbox-redis')}:{os.environ.get('REDIS_PORT', '6379')}/{os.environ.get('REDIS_DATABASE', '0')}\"\n" + with open(f"{netboxSettingsPyDir}/local_settings.py", 'w') as f: + f.write(localSettingsPyContents) + except Exception as e: + logging.error(f"{type(e).__name__} writing local_settings.py: {e}") + if os.path.isdir(args.customPluginsDir) and os.path.isfile(os.path.join(args.netboxConfigDir, 'plugins.py')): # get a list of what packages/plugins already installed (package names and versions in a dict) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index d837369dc..2de0a8b0f 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -194,7 +194,7 @@ http { # extracted file download location ~* ^/extracted-files\b(.*) { include /etc/nginx/nginx_auth_rt.conf; - proxy_pass http://extracted-file-http-server$1; + proxy_pass http://extracted-file-http-server$1$is_args$args; proxy_redirect off; proxy_set_header Host file-monitor.malcolm.local; } @@ -204,6 +204,7 @@ http { include /etc/nginx/nginx_auth_rt.conf; include /etc/nginx/nginx_system_resolver.conf; set $upstream $1:8006; + # TODO: check, do i need is_args/args here? rewrite ^/hh-extracted-files/([a-zA-Z0-9-\.]+)(.*)$ $2 break; proxy_pass https://$upstream; proxy_ssl_verify off; diff --git a/scripts/control.py b/scripts/control.py index e18d6395e..2430685c8 100755 --- a/scripts/control.py +++ b/scripts/control.py @@ -1072,7 +1072,7 @@ def start(): BoundPath("zeek", "/zeek/extract_files", False, None, None), BoundPath("zeek", "/zeek/upload", False, None, None), BoundPath("zeek", "/opt/zeek/share/zeek/site/custom", False, None, None), - BoundPath("zeek", "/opt/zeek/share/zeek/site/intel", False, ["MISP", "STIX"], None), + BoundPath("zeek", "/opt/zeek/share/zeek/site/intel", False, ["Mandiant", "MISP", "STIX"], None), BoundPath("zeek-live", "/zeek/live", False, ["spool"], None), BoundPath( "filebeat", "/zeek", False, ["processed", "current", "live", "extract_files", "upload"], None @@ -1341,7 +1341,7 @@ def authSetup(): 'netbox', "(Re)generate internal passwords for NetBox", False, - args.cmdAuthSetupNonInteractive and args.authGenNetBoxPasswords, + (not args.cmdAuthSetupNonInteractive) or args.authGenNetBoxPasswords, [], ), ( @@ -2346,7 +2346,8 @@ def main(): try: parser.error = parser.exit args = parser.parse_args() - except SystemExit: + except SystemExit as e: + eprint(f'Invalid arguments: {e}') parser.print_help() exit(2) diff --git a/scripts/install.py b/scripts/install.py index 52879e881..4116fcff7 100755 --- a/scripts/install.py +++ b/scripts/install.py @@ -3365,7 +3365,14 @@ def tweak_system_files(self): '/etc/security/limits.d/limits.conf', '', '/etc/security/limits.d/limits.conf increases the allowed maximums for file handles and memlocked segments', - ['* soft nofile 65535', '* hard nofile 65535', '* soft memlock unlimited', '* hard memlock unlimited'], + [ + '* soft nofile 65535', + '* hard nofile 65535', + '* soft memlock unlimited', + '* hard memlock unlimited', + '* soft nproc 262144', + '* hard nproc 524288', + ], ), ] @@ -3760,7 +3767,7 @@ def main(): metavar='', type=str, default='', - help='Malcolm container images .tar.gz file for installation', + help='Malcolm container images .tar.xz file for installation', ) runtimeOptionsArgGroup = parser.add_argument_group('Runtime options') @@ -4014,7 +4021,7 @@ def main(): required=False, metavar='', type=int, - default=None, + default=0, help='Number of Logstash workers (e.g., 4, 8, etc.)', ) opensearchArgGroup.add_argument( @@ -4569,7 +4576,8 @@ def main(): try: parser.error = parser.exit args = parser.parse_args() - except SystemExit: + except SystemExit as e: + eprint(f'Invalid arguments: {e}') parser.print_help() exit(2) diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh index 5180211d8..8d39f2467 100755 --- a/scripts/malcolm_appliance_packager.sh +++ b/scripts/malcolm_appliance_packager.sh @@ -29,12 +29,17 @@ if ! (type "$REALPATH" && type "$DIRNAME") > /dev/null; then fi SCRIPT_PATH="$($DIRNAME $($REALPATH -e "${BASH_SOURCE[0]}"))" pushd "$SCRIPT_PATH/.." >/dev/null 2>&1 +MALCOLM_CONTAINER_RUNTIME="${MALCOLM_CONTAINER_RUNTIME:-docker}" CURRENT_REV_SHA="$(git rev-parse --short --verify HEAD)" if [ -z "$CURRENT_REV_SHA" ]; then CURRENT_REV_TAG="$(date +%Y.%m.%d_%H:%M:%S)" else - CURRENT_REV_DATE="$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y%m%d_%H%M%S)" + if [[ "$(uname -s)" == 'Darwin' ]]; then + CURRENT_REV_DATE="$(git log -1 --format="%at" | xargs -I{} date -r {} +%Y%m%d_%H%M%S)" + else + CURRENT_REV_DATE="$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y%m%d_%H%M%S)" + fi if [ -z "$CURRENT_REV_DATE" ]; then CURRENT_REV_TAG="$(date +%Y.%m.%d_%H:%M:%S)" fi @@ -93,6 +98,7 @@ if mkdir "$DESTDIR"; then mkdir $VERBOSE -p "$DESTDIR/zeek-logs/processed/" mkdir $VERBOSE -p "$DESTDIR/zeek-logs/upload/" mkdir $VERBOSE -p "$DESTDIR/zeek/custom/" + mkdir $VERBOSE -p "$DESTDIR/zeek/intel/Mandiant/" mkdir $VERBOSE -p "$DESTDIR/zeek/intel/MISP/" mkdir $VERBOSE -p "$DESTDIR/zeek/intel/STIX/" @@ -143,7 +149,11 @@ if mkdir "$DESTDIR"; then cp $VERBOSE "$SCRIPT_PATH/malcolm_kubernetes.py" "$RUN_PATH/" cp $VERBOSE "$SCRIPT_PATH/malcolm_utils.py" "$RUN_PATH/" - tar $VERBOSE --numeric-owner --owner=0 --group=0 -czf "$DESTNAME" "./$(basename $DESTDIR)/" + if [[ "$(uname -s)" == "Darwin" ]]; then + tar $VERBOSE -czf "$DESTNAME" "./$(basename $DESTDIR)/" + else + tar $VERBOSE --numeric-owner --owner=0 --group=0 -czf "$DESTNAME" "./$(basename $DESTDIR)/" + fi echo "Packaged Malcolm to \"$DESTNAME\"" unset CONFIRMATION @@ -154,7 +164,11 @@ if mkdir "$DESTDIR"; then echo "This might take a few minutes..." DESTNAMEIMAGES="$RUN_PATH/$(basename $DESTDIR)_images.tar.xz" IMAGES=( $(grep image: $DESTDIR/docker-compose.yml | awk '{print $2}' | sort -u) ) - docker save "${IMAGES[@]}" | xz -1 > "$DESTNAMEIMAGES" + if [[ "$MALCOLM_CONTAINER_RUNTIME" == "podman" ]]; then + $MALCOLM_CONTAINER_RUNTIME save --multi-image-archive --format docker-archive "${IMAGES[@]}" | xz -1 > "$DESTNAMEIMAGES" + else + $MALCOLM_CONTAINER_RUNTIME save "${IMAGES[@]}" | xz -1 > "$DESTNAMEIMAGES" + fi echo "Packaged Malcolm container images to \"$DESTNAMEIMAGES\"" echo "" fi diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index 883b7e8f8..923827a72 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -3,7 +3,7 @@ "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", "instance_arch": "x86_64", - "malcolm_tag": "v24.10.1", + "malcolm_tag": "v24.11.0", "malcolm_repo": "idaholab/Malcolm", "malcolm_uid": "1000", "ssh_username": "ec2-user", diff --git a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh index 126bce31a..8a101ec2e 100755 --- a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh +++ b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh @@ -32,7 +32,7 @@ fi # -u UID (user UID, e.g., 1000) VERBOSE_FLAG= MALCOLM_REPO=${MALCOLM_REPO:-idaholab/Malcolm} -MALCOLM_TAG=${MALCOLM_TAG:-v24.10.1} +MALCOLM_TAG=${MALCOLM_TAG:-v24.11.0} [[ -z "$MALCOLM_UID" ]] && ( [[ $EUID -eq 0 ]] && MALCOLM_UID=1000 || MALCOLM_UID="$(id -u)" ) while getopts 'vr:t:u:' OPTION; do case "$OPTION" in diff --git a/shared/bin/extracted_files_http_server.py b/shared/bin/extracted_files_http_server.py index 7071f878d..8842add80 100755 --- a/shared/bin/extracted_files_http_server.py +++ b/shared/bin/extracted_files_http_server.py @@ -24,6 +24,7 @@ from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler from stat import S_IFREG from stream_zip import ZIP_32, stream_zip +from urllib.parse import urlparse, parse_qs from malcolm_utils import ( eprint, @@ -49,12 +50,6 @@ malcolm_forward_header = 'X-Malcolm-Forward' -################################################################################################### -# a function for performing "natural" (case insensitive) sort -def natural_sort_key(s, _nsre=re.compile('([0-9]+)')): - return [int(text) if text.isdigit() else text.lower() for text in _nsre.split(s)] - - ################################################################################################### # return the names and flags for Zipping a list of files def LocalFilesForZip(names): @@ -129,6 +124,51 @@ def do_GET(self): with div(cls='container'): h1(pageTitle, cls='mb-5', style='text-align: center') with div(cls='container').add(div(cls="row")).add(div(cls="col-lg-12")): + + # parse the query parameters to get the page number + parsedUrl = urlparse(self.path) + queryParams = parse_qs(parsedUrl.query) + page = int(queryParams.get('page', ['1'])[0]) + page = max(page, 1) + + # now get # of elements to display per page + elements = int(queryParams.get('elements', ['1'])[0]) + elements = max(elements, 50) + itemsPerPage = elements + + items = [] + + for dirpath, dirnames, filenames in os.walk(fullpath): + # list directories first + for dirname in sorted(dirnames, key=str.casefold): + try: + child = os.path.join(dirpath, dirname) + if args.links or (not os.path.islink(child)): + items.append(('dir', dirname, child)) + except Exception as e: + eprint(f'Error with directory "{dirname}"": {e}') + # list files + for filename in sorted(filenames, key=str.casefold): + try: + child = os.path.join(dirpath, filename) + if args.links or (not os.path.islink(child)): + items.append(('file', filename, child)) + except Exception as e: + eprint(f'Error with file "{filename}"": {e}') + # our "walk" is not recursive right now, we only need to go one level deep + break + + totalItems = len(items) + totalPages = (totalItems + itemsPerPage - 1) // itemsPerPage + + # ensure the page number is within valid range + page = min(page, totalPages) if totalPages > 0 else 1 + + # get items for the current page + startIndex = (page - 1) * itemsPerPage + endIndex = startIndex + itemsPerPage + itemsOnPage = items[startIndex:endIndex] + with table(cls='table-bordered', width='100%').add(tbody()): # header row t = tr(style="text-align: center") @@ -155,150 +195,173 @@ def do_GET(self): if showMalcolmCols: t.add(th(), th(), th()) - # content rows (files and directories) - for dirpath, dirnames, filenames in os.walk(fullpath): - # list directories first - for dirname in sorted(dirnames, key=natural_sort_key): - try: - child = os.path.join(dirpath, dirname) - if args.links or (not os.path.islink(child)): - t = tr() - t.add( - td(a(dirname, href=f'{dirname}/')), - td("Directory"), - td(''), - ) - if showMalcolmCols: - t.add(th(), th(), th()) - except Exception as e: - eprint(f'Error with directory "{dirname}"": {e}') - - # list files after directories - for filename in sorted(filenames, key=natural_sort_key): - try: - child = os.path.join(dirpath, filename) - if args.links or (not os.path.islink(child)): - t = tr() - - # calculate some of the stuff for representing Malcolm files - timestamp = None - timestampStr = '' - timestampStartFilterStr = '' - fmatch = None - fsource = '' - fids = list() - if showMalcolmCols: - # determine if filename is in a pattern we recognize - fmatch = carvedFileRegex.search(filename) - if fmatch is None: - fmatch = carvedFileRegexAlt.search(filename) - if fmatch is not None: - # format timestamp as ISO date/time - timestampStr = fmatch.groupdict().get('timestamp', '') - try: - timestamp = datetime.strptime(timestampStr, '%Y%m%d%H%M%S') - timestampStr = timestamp.isoformat() - timestampStartFilterStr = ( - (timestamp - timedelta(days=1)) - .isoformat() - .split('.')[0] - ) - except Exception as te: - if timestampStr: - eprint(f'Error with time "{str(timestampStr)}": {te}') - # put UIDs and FUIDs into a single event.id-filterable column - fids = list( - [ - x - for x in [ - fmatch.groupdict().get('uid', ''), - fmatch.groupdict().get('fuid', ''), - ] - if x and x != 'unknown' - ] + # content rows + for itemType, filename, child in itemsOnPage: + try: + if itemType == 'dir': + t = tr() + t.add( + td(a(filename, href=f'{filename}/?page=1&elements={elements}')), + td("Directory"), + td(''), + ) + + if showMalcolmCols: + t.add(th(), th(), th()) + elif itemType == 'file': + t = tr() + + # calculate some of the stuff for representing Malcolm files + timestamp = None + timestampStr = '' + timestampStartFilterStr = '' + fmatch = None + fsource = '' + fids = list() + if showMalcolmCols: + # determine if filename is in a pattern we recognize + fmatch = carvedFileRegex.search(filename) + if fmatch is None: + fmatch = carvedFileRegexAlt.search(filename) + if fmatch is not None: + # format timestamp as ISO date/time + timestampStr = fmatch.groupdict().get('timestamp', '') + try: + timestamp = datetime.strptime(timestampStr, '%Y%m%d%H%M%S') + timestampStr = timestamp.isoformat() + timestampStartFilterStr = ( + (timestamp - timedelta(days=1)).isoformat().split('.')[0] ) - # massage source a little bit (remove '' and handle - # 'XOR decrypted from...') - fsource = fmatch.groupdict().get('source', '') - if fsource == '': - fsource = '' - elif xorMatch := xorRegex.search(fsource): - fsource = xorMatch.groupdict().get('source', '') - fids.append(xorMatch.groupdict().get('fuid', '')) - - # only request mime type for files if specified in arguments - fileinfo = ( - magic.from_file(os.path.realpath(child), mime=True) - if args.magic - else os.path.splitext(filename)[1] - ) - - # show filename, file type (with link to IANA if MIME type is shown), and file size - t.add( + except Exception as te: + if timestampStr: + eprint(f'Error with time "{str(timestampStr)}": {te}') + # put UIDs and FUIDs into a single event.id-filterable column + fids = list( + [ + x + for x in [ + fmatch.groupdict().get('uid', ''), + fmatch.groupdict().get('fuid', ''), + ] + if x and x != 'unknown' + ] + ) + # massage source a little bit (remove '' and handle + # 'XOR decrypted from...') + fsource = fmatch.groupdict().get('source', '') + if fsource == '': + fsource = '' + elif xorMatch := xorRegex.search(fsource): + fsource = xorMatch.groupdict().get('source', '') + fids.append(xorMatch.groupdict().get('fuid', '')) + + # only request mime type for files if specified in arguments + fileinfo = ( + magic.from_file(os.path.realpath(child), mime=True) + if args.magic + else os.path.splitext(filename)[1] + ) + + # show filename, file type (with link to IANA if MIME type is shown), and file size + t.add( + td( + a( + ( + (filename[:fnameDispLen] + '...') + if len(filename) > fnameDispLen + else filename + ), + href=f'{filename}', + ), + title=filename, + ), + ( td( a( - ( - (filename[:fnameDispLen] + '...') - if len(filename) > fnameDispLen - else filename - ), - href=f'{filename}', + fileinfo, + href=f'https://www.iana.org/assignments/media-types/{fileinfo}', + target="_blank", ), - title=filename, - ), - ( + ) + if args.magic + else td(fileinfo) + ), + td(sizeof_fmt(os.path.getsize(child)), style="text-align: right"), + ) + + # show special malcolm columns if requested + if showMalcolmCols: + if fmatch is not None: + # list carve source, IDs, and timestamp + t.add( td( - a( - fileinfo, - href=f'https://www.iana.org/assignments/media-types/{fileinfo}', - target="_blank", - ), - ) - if args.magic - else td(fileinfo) - ), - td(sizeof_fmt(os.path.getsize(child)), style="text-align: right"), - ) - - # show special malcolm columns if requested - if showMalcolmCols: - if fmatch is not None: - # list carve source, IDs, and timestamp - t.add( - td( - fsource, - style="text-align: center", - ), - td( - [ - a( - fid, - href=f'/arkime/idark2dash/filter?start={timestampStartFilterStr}&stop={tomorrowStr}&field=event.id&value={fid}', - target="_blank", - ) - for fid in fids - ], - style="text-align: center", - ), - td( - ( - timestamp.strftime("%Y-%m-%d %H:%M:%S") - if timestamp - else timestampStr - ), - title=timestampStr, - style="text-align: center", + fsource, + style="text-align: center", + ), + td( + [ + a( + fid, + href=f'/arkime/idark2dash/filter?start={timestampStartFilterStr}&stop={tomorrowStr}&field=event.id&value={fid}', + target="_blank", + ) + for fid in fids + ], + style="text-align: center", + ), + td( + ( + timestamp.strftime("%Y-%m-%d %H:%M:%S") + if timestamp + else timestampStr ), - ) - else: - # file name format was not recognized, so extra columns are empty - t.add(th(), th(), th()) - - except Exception as e: - eprint(f'Error with file "{filename}": {e}') + title=timestampStr, + style="text-align: center", + ), + ) + else: + # file name format was not recognized, so extra columns are empty + t.add(th(), th(), th()) - # our "walk" is not recursive right now, we only need to go one level deep - break + except Exception as e: + eprint(f'Error with file "{filename}": {e}') + + # pagination controls + br() + with div( + cls='pagination', + style='text-align: center; display: flex; justify-content: center; padding: 0;', + ): + with ul( + cls='pagination-list', + style='display: flex; list-style: none; justify-content: center; padding: 0;', + ): + # previous page link + if page > 1: + prevPageUrl = f'?page={page - 1}&elements={elements}' + li( + a( + f'Previous ({page - 1})', + href=prevPageUrl, + cls='page-link', + ), + cls='page-item', + ) + else: + li(span('Previous', cls='page-link disabled'), cls='page-item') + + # add a space between text + li(' ', cls='page-item spacer', style='width: 10px;') + + # next page link + if page < totalPages: + nextPageUrl = f'?page={page + 1}&elements={elements}' + li( + a(f'Next ({page + 1} of {totalPages})', href=nextPageUrl, cls='page-link'), + cls='page-item', + ) + else: + li(span('Next', cls='page-link disabled'), cls='page-item') # footer decoration with footer(cls='footer bg-light').add(div(cls='container')).add(div(cls='row')): diff --git a/shared/bin/suricata_config_populate.py b/shared/bin/suricata_config_populate.py index c9b356852..b270c89d2 100755 --- a/shared/bin/suricata_config_populate.py +++ b/shared/bin/suricata_config_populate.py @@ -1282,7 +1282,7 @@ def main(): tmpLogDir, '-T', ], - debug=args.verbose > logging.DEBUG, + debug=args.verbose <= logging.DEBUG, logger=logging, ) logging.info(f'suricata configuration test returned {script_return_code}') diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh index 021a7b1a4..2ecb92b7c 100755 --- a/shared/bin/zeek_install_plugins.sh +++ b/shared/bin/zeek_install_plugins.sh @@ -63,10 +63,15 @@ function zkg_install_github_repo() { export "$ENV" done fi + EXTRA_ZKG_PARAMS=() + if [[ "$REPO_URL" =~ "zeek-kafka" ]]; then + EXTRA_ZKG_PARAMS+=( --user-var ) + EXTRA_ZKG_PARAMS+=( LIBRDKAFKA_ROOT=/usr/include/librdkafka ) + fi if [[ -n $REPO_LATEST_RELEASE ]]; then - zkg install --nodeps --force --skiptests --version "$REPO_LATEST_RELEASE" "$REPO_URL" + zkg install --nodeps --force --skiptests "${EXTRA_ZKG_PARAMS[@]}" --version "$REPO_LATEST_RELEASE" "$REPO_URL" else - zkg install --nodeps --force --skiptests "$REPO_URL" + zkg install --nodeps --force --skiptests "${EXTRA_ZKG_PARAMS[@]}" "$REPO_URL" fi fi } @@ -121,12 +126,13 @@ ZKG_GITHUB_URLS=( "https://github.com/corelight/zeek-xor-exe-plugin|master" "https://github.com/corelight/zerologon" "https://github.com/cybera/zeek-sniffpass" - "https://github.com/piercema/ja4" + "https://github.com/FoxIO-LLC/ja4|main" "https://github.com/mmguero-dev/bzar" "https://github.com/ncsa/bro-is-darknet" "https://github.com/ncsa/bro-simple-scan" "https://github.com/precurse/zeek-httpattacks" "https://github.com/mmguero-dev/GQUIC_Protocol_Analyzer" + "https://github.com/SeisoLLC/zeek-kafka" "https://github.com/zeek/spicy-tftp" "https://github.com/zeek/spicy-zip" ) diff --git a/shared/bin/zeek_intel_from_threat_feed.py b/shared/bin/zeek_intel_from_threat_feed.py index ca6153f1f..410ff19e4 100755 --- a/shared/bin/zeek_intel_from_threat_feed.py +++ b/shared/bin/zeek_intel_from_threat_feed.py @@ -12,7 +12,9 @@ import argparse import logging import os +import re import sys +import yaml import zeek_threat_feed_utils import malcolm_utils @@ -32,6 +34,7 @@ def main(): 'Outputs a Zeek intelligence framework file from various formats used to represent threat information:', ' - "Indicator" objects in STIX™ v2.0/v2.1 JSON files', ' - MISP attributes or core format JSON files', + ' - Indicators from Mantiant Threat Intelligence', '', 'See:', ' - Malcolm documentation: https://idaholab.github.io/Malcolm/docs/zeek-intel.html#ZeekIntel', @@ -45,6 +48,7 @@ def main(): ' - MISP default feeds: https://www.misp-project.org/feeds/', ' - Managing MISP feeds: https://misp.gitbooks.io/misp-book/content/managing-feeds/', ' - Expand MISP usage: https://github.com/idaholab/Malcolm/issues/336', + ' - Mandiant Threat Intelligence Indicators API: https://docs.mandiant.com/home/mati-threat-intelligence-api-v4#tag/Indicators' '', 'Note: The Zeek intelligence framework only supports simple indicators matched against a single value.', 'The STIX™ standard can express more complex indicators that cannot be expressed with Zeek intelligence items.', @@ -103,7 +107,7 @@ def main(): nargs='*', type=str, default=None, - help="Read --input arguments from a local or external file (one per line)", + help="Read --input arguments from a local or external file (one per line, or YAML definitions)", ) parser.add_argument( '-o', @@ -148,6 +152,7 @@ def main(): if args.input is None: args.input = [] + yamlInputs = [] since = ( ParseDate(args.since).astimezone(UTCTimeZone) if (args.since is not None) and (len(args.since) > 0) else None ) @@ -164,9 +169,23 @@ def main(): for infileArg in args.inputFile: try: if os.path.isfile(infileArg): - # read inputs from local file - with open(infileArg) as f: - args.input.extend(f.read().splitlines()) + # read inputs from local file (delimited lines or YAML file) + infileParts = os.path.splitext(infileArg) + if re.search(r"\.ya?ml$", infileParts[1], re.IGNORECASE): + with open(infileArg, 'r') as f: + inputParams = yaml.safe_load(f) + if inputParams: + if isinstance(inputParams, dict): + yamlInputs.append(inputParams) + elif isinstance(inputParams, list): + yamlInputs.extend(inputParams) + else: + logging.error( + f"Connection parameters of type '{type(inputParams).__name__}' are not supported" + ) + else: + with open(infileArg) as f: + args.input.extend(f.read().splitlines()) elif '://' in infileArg: # download from URL and read input from remote file @@ -183,17 +202,22 @@ def main(): else: logging.warning(f"File '{infileArg}' not found") + except Exception as e: - logging.warning(f"{type(e).__name__} for '{infileArg}': {e}") + logging.error(f"{type(e).__name__} for '{infileArg}': {e}") # deduplicate input sources seenInput = {} args.input = [seenInput.setdefault(x, x) for x in args.input if x not in seenInput] - logging.debug(f"Input: {args.input}") # we'll queue and then process all of the input arguments in workers inputQueue = deque() - inputQueue.extend(args.input) + if args.input: + inputQueue.extend(args.input) + if yamlInputs: + inputQueue.extend(yamlInputs) + logging.debug(f"Inputs: {list(inputQueue)}") + workerThreadCount = malcolm_utils.AtomicInt(value=0) ThreadPool( args.threads, diff --git a/shared/bin/zeek_intel_setup.sh b/shared/bin/zeek_intel_setup.sh index f15e4e8d3..35b1476e6 100755 --- a/shared/bin/zeek_intel_setup.sh +++ b/shared/bin/zeek_intel_setup.sh @@ -47,7 +47,7 @@ if mkdir -- "$LOCK_DIR" 2>/dev/null; then done < <(echo "${CONFIG_MAP_DIR:-configmap;secretmap}" | tr ';' '\n') rsync --recursive --delete --delete-excluded "${EXCLUDES[@]}" "${INTEL_PRESEED_DIR}"/ "${INTEL_DIR}"/ - mkdir -p "${INTEL_DIR}"/MISP "${INTEL_DIR}"/STIX || true + mkdir -p "${INTEL_DIR}"/MISP "${INTEL_DIR}"/STIX "${INTEL_DIR}"/Mandiant || true fi # create directive to @load every subdirectory in /opt/zeek/share/zeek/site/intel @@ -75,34 +75,40 @@ EOF # this directory contains STIX JSON files we'll need to convert to zeek intel files then load while IFS= read -r line; do THREAT_JSON_FILES+=( "$line" ) - done < <( find "${INTEL_DIR}/${DIR}" -type f ! -name ".*" 2>/dev/null ) + done < <( find "${INTEL_DIR}/${DIR}" \( -type f -a ! -name ".*" -a ! -name "taxii.yaml" \) 2>/dev/null ) elif [[ "${DIR}" == "./MISP" ]]; then # this directory contains MISP JSON files we'll need to convert to zeek intel files then load while IFS= read -r line; do THREAT_JSON_FILES+=( "$line" ) - done < <( find "${INTEL_DIR}/${DIR}" -type f ! -name ".*" ! -name "manifest.json" ! -name "hashes.csv" 2>/dev/null ) + done < <( find "${INTEL_DIR}/${DIR}" \( -type f -a ! -name ".*" -a ! -name "misp.yaml" -a ! -name "manifest.json" -a ! -name "hashes.csv" \) 2>/dev/null ) elif [[ -f "${DIR}"/__load__.zeek ]]; then # this intel feed has its own load directive and should take care of itself echo "@load ${DIR}" >> ./__load__.zeek."${INSTANCE_UID}" + else - # this directory contains "loose" intel files we'll need to load explicitly + # this custom directory contains "loose" intel files we'll need to load explicitly while IFS= read -r line; do LOOSE_INTEL_FILES+=( "$line" ) - done < <( find "${INTEL_DIR}/${DIR}" -type f ! -name ".*" 2>/dev/null ) + done < <( find "${INTEL_DIR}/${DIR}" \( -type f -a ! -name ".*" -a ! -name "*.yaml" \) 2>/dev/null ) fi done - # process STIX and MISP inputs by converting them to Zeek intel format - if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./MISP/.misp_input.txt ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then + # process STIX/MISP/Mandiant inputs by converting them to Zeek intel format + if ( (( ${#THREAT_JSON_FILES[@]} )) || [[ -r ./STIX/.stix_input.txt ]] || [[ -r ./STIX/taxii.yaml ]] || [[ -r ./MISP/.misp_input.txt ]] || [[ -r ./MISP/misp.yaml ]] || [[ -r ./Mandiant/mandiant.yaml ]] ) && [[ -x "${THREAT_FEED_TO_ZEEK_SCRIPT}" ]]; then "${THREAT_FEED_TO_ZEEK_SCRIPT}" \ --ssl-verify ${ZEEK_INTEL_FEED_SSL_CERTIFICATE_VERIFICATION} \ --since "${ZEEK_INTEL_FEED_SINCE}" \ --threads ${ZEEK_INTEL_REFRESH_THREADS} \ --output ./.threat_autogen.zeek."${INSTANCE_UID}" \ --input "${THREAT_JSON_FILES[@]}" \ - --input-file ./STIX/.stix_input.txt ./MISP/.misp_input.txt + --input-file \ + ./STIX/.stix_input.txt \ + ./STIX/taxii.yaml \ + ./MISP/.misp_input.txt \ + ./MISP/misp.yaml \ + ./Mandiant/mandiant.yaml if [[ $? -eq 0 ]]; then rm -f ./.threat_autogen.zeek.old mv --backup=simple --suffix=.old ./.threat_autogen.zeek."${INSTANCE_UID}" ./.threat_autogen.zeek diff --git a/shared/bin/zeek_threat_feed_utils.py b/shared/bin/zeek_threat_feed_utils.py index 1341a243b..81d1e0697 100644 --- a/shared/bin/zeek_threat_feed_utils.py +++ b/shared/bin/zeek_threat_feed_utils.py @@ -11,6 +11,7 @@ from collections.abc import Iterable from contextlib import contextmanager, nullcontext from datetime import datetime +from dateutil.relativedelta import relativedelta from multiprocessing import RawValue from pymisp import MISPEvent, MISPAttribute, PyMISP from pytz import utc as UTCTimeZone @@ -28,10 +29,13 @@ from taxii2client.v21 import Server as TaxiiServer_v21 from threading import Lock from time import sleep, mktime +from types import GeneratorType, FunctionType, LambdaType from typing import Tuple, Union from urllib.parse import urljoin, urlparse from logging import DEBUG as LOGGING_DEBUG +import copy import json +import mandiant_threatintel import os import re import requests @@ -39,7 +43,7 @@ from malcolm_utils import eprint, base64_decode_if_prefixed, LoadStrIfJson, LoadFileIfJson, isprivateip -# keys for dict returned by map_stix_indicator_to_zeek for Zeek intel file fields +# keys for dict returned by map_*_indicator_to_zeek for Zeek intel file fields ZEEK_INTEL_INDICATOR = 'indicator' ZEEK_INTEL_INDICATOR_TYPE = 'indicator_type' ZEEK_INTEL_META_SOURCE = 'meta.source' @@ -53,12 +57,20 @@ ZEEK_INTEL_CIF_FIRSTSEEN = 'meta.cif_firstseen' ZEEK_INTEL_CIF_LASTSEEN = 'meta.cif_lastseen' +ZEEK_INTEL_WORKER_THREADS_DEFAULT = 2 + TAXII_INDICATOR_FILTER = {'type': 'indicator'} TAXII_PAGE_SIZE = 50 MISP_PAGE_SIZE_ATTRIBUTES = 500 MISP_PAGE_SIZE_EVENTS = 10 -ZEEK_INTEL_WORKER_THREADS_DEFAULT = 2 - +MANDIANT_PAGE_SIZE_DEFAULT = 1000 +MANDIANT_MINIMUM_MSCORE_DEFAULT = 60 +MANDIANT_EXCLUDE_OSINT_DEFAULT = False +MANDIANT_INCLUDE_CAMPAIGNS_DEFAULT = False +MANDIANT_INCLUDE_REPORTS_DEFAULT = False +MANDIANT_INCLUDE_THREAT_RATING_DEFAULT = False +MANDIANT_INCLUDE_MISP_DEFAULT = True +MANDIANT_INCLUDE_CATEGORY_DEFAULT = True # See the documentation for the Zeek INTEL framework [1] and STIX-2 cyber observable objects [2] # [1] https://docs.zeek.org/en/stable/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type @@ -115,6 +127,16 @@ "x509-fingerprint-sha1": "CERT_HASH", } +# See the documentation for the Zeek INTEL framework [1] and Mandiant threat intel API [2] +# [1] https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type +# [2] https://docs.mandiant.com/home/mati-threat-intelligence-api-v4#tag/Indicators +MANDIANT_ZEEK_INTEL_TYPE_MAP = { + mandiant_threatintel.FQDNIndicator: 'DOMAIN', + mandiant_threatintel.URLIndicator: 'URL', + mandiant_threatintel.IPIndicator: 'ADDR', + mandiant_threatintel.MD5Indicator: 'FILE_HASH', +} + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) @@ -155,7 +177,7 @@ def download_to_file(url, session=None, local_filename=None, chunk_bytes=4096, s f.write(chunk) fExists = os.path.isfile(tmpDownloadedFileSpec) fSize = os.path.getsize(tmpDownloadedFileSpec) - if logger is not None: + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): logger.debug( f"Download of {url} to {tmpDownloadedFileSpec} {'succeeded' if fExists else 'failed'} ({fSize} bytes)" ) @@ -168,6 +190,112 @@ def download_to_file(url, session=None, local_filename=None, chunk_bytes=4096, s return None +def mandiant_indicator_as_json_str(indicator, skip_attr_map={}): + if indicator and indicator._api_response: + return json.dumps(indicator._api_response) + else: + return 'unknown indicator' + + +def map_mandiant_indicator_to_zeek( + indicator: mandiant_threatintel.APIResponse, + skip_attr_map={}, + logger=None, +) -> Union[Tuple[defaultdict], None]: + """ + Maps a Mandiant threat intelligence indicator object to Zeek intel items + @see https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html#type-Intel::Type + @param indicator The indicator object (mandiant_threatintel.APIResponse) to convert + @return a list containing the Zeek intel dict(s) from the indicator + """ + results = [] + + # get matching Zeek intel type + if zeek_type := MANDIANT_ZEEK_INTEL_TYPE_MAP.get(type(indicator), None): + + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): + logger.debug(mandiant_indicator_as_json_str(indicator, skip_attr_map=skip_attr_map)) + + zeekItem = defaultdict(lambda: '-') + tags = [] + sources = [] + + zeekItem[ZEEK_INTEL_INDICATOR_TYPE] = "Intel::" + zeek_type + + if hasattr(indicator, 'id'): + zeekItem[ZEEK_INTEL_META_DESC] = indicator.id + zeekItem[ZEEK_INTEL_CIF_DESCRIPTION] = zeekItem[ZEEK_INTEL_META_DESC] + zeekItem[ZEEK_INTEL_META_URL] = f'https://advantage.mandiant.com/indicator/{indicator.id}' + if hasattr(indicator, 'mscore'): + zeekItem[ZEEK_INTEL_CIF_CONFIDENCE] = str(round(indicator.mscore / 10)) + if hasattr(indicator, 'first_seen'): + zeekItem[ZEEK_INTEL_CIF_FIRSTSEEN] = str(mktime(indicator.first_seen.timetuple())) + if hasattr(indicator, 'last_seen'): + zeekItem[ZEEK_INTEL_CIF_LASTSEEN] = str(mktime(indicator.last_seen.timetuple())) + if hasattr(indicator, 'sources'): + sources.extend(list({entry['source_name'] for entry in indicator.sources if 'source_name' in entry})) + if categories := list( + { + category + for item in indicator.sources + if 'category' in item and item['category'] + for category in item['category'] + } + ): + tags.extend(categories) + + if hasattr(indicator, 'misp'): + if trueMispAttrs := [key for key, value in indicator.misp.items() if value]: + tags.extend(trueMispAttrs) + + if tags: + zeekItem[ZEEK_INTEL_CIF_TAGS] = ','.join([x.replace(',', '\\x2c') for x in tags]) + + # The MD5Indicator class can actually have multiple types of hashes, + # and we want to create a zeek intel item for each. I'm accessing + # the underlying API response directly here (rather than through getattr) + # to avoid extra GET requests to the API attempting to find a value + # that didn't come with the initial request. + # Performance-wise, if we didn't get it with the indicator object in + # the first place it's not something we need to make an entire extra + # network communication to attempt. + if ( + isinstance(indicator, mandiant_threatintel.MD5Indicator) + and indicator._api_response + and (hashes := indicator._api_response.get('associated_hashes', [])) + ): + for hashish in hashes: + if hashVal := hashish.get('value', None): + tmpItem = copy.deepcopy(zeekItem) + tmpItem[ZEEK_INTEL_INDICATOR] = hashVal + if newId := hashish.get('id', None): + tmpItem[ZEEK_INTEL_META_URL] = f'https://advantage.mandiant.com/indicator/{newId}' + if ZEEK_INTEL_META_URL in tmpItem: + sources.append(tmpItem[ZEEK_INTEL_META_URL]) + if sources: + tmpItem[ZEEK_INTEL_META_SOURCE] = '\\x7c'.join([x.replace(',', '\\x2c') for x in sources]) + results.append(tmpItem) + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): + logger.debug(tmpItem) + + elif hasattr(indicator, 'value') and (val := indicator.value): + # handle other types besides the file hash + zeekItem[ZEEK_INTEL_INDICATOR] = val + if ZEEK_INTEL_META_URL in zeekItem: + sources.append(zeekItem[ZEEK_INTEL_META_URL]) + if sources: + zeekItem[ZEEK_INTEL_META_SOURCE] = '\\x7c'.join([x.replace(',', '\\x2c') for x in sources]) + results.append(zeekItem) + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): + logger.debug(zeekItem) + + else: + if logger is not None: + logger.warning(f"No matching Zeek type found for Mandiant indicator type '{indicator.__class__.__name__}'") + + return results + + def stix_pattern_from_str(indicator_type: type, pattern_str: str) -> Union[STIX_Pattern_v21, STIX_Pattern_v20, None]: """ Creates a stix2patterns.v20.pattern.Pattern (STIX_Pattern_v20) or a @@ -300,7 +428,7 @@ def map_stix_indicator_to_zeek( ) return None - if logger is not None: + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): logger.debug(indicator) results = [] @@ -329,7 +457,7 @@ def map_stix_indicator_to_zeek( zeekItem = defaultdict(lambda: '-') zeekItem[ZEEK_INTEL_META_SOURCE] = ( - ','.join([x.replace(',', '\\x2c') for x in source]) + '\\x7c'.join([x.replace(',', '\\x2c') for x in source]) if source is not None and len(source) > 0 else str(indicator.id) ) @@ -339,6 +467,7 @@ def map_stix_indicator_to_zeek( zeekItem[ZEEK_INTEL_META_DESC] = '. '.join( [x for x in [indicator.get('name', None), indicator.get('description', None)] if x is not None] ) + zeekItem[ZEEK_INTEL_CIF_DESCRIPTION] = zeekItem[ZEEK_INTEL_META_DESC] # some of these are from CFM, what the heck... # if 'description' in indicator: # "description": "severity level: Low\n\nCONFIDENCE: High", @@ -354,7 +483,7 @@ def map_stix_indicator_to_zeek( # TODO: confidence? results.append(zeekItem) - if logger is not None: + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): logger.debug(zeekItem) return results @@ -375,7 +504,7 @@ def map_misp_attribute_to_zeek( @param attribute The MISPAttribute to convert @return a list containing the Zeek intel dict(s) from the MISPAttribute object """ - if logger is not None: + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): logger.debug(attribute.to_json()) results = [] @@ -413,9 +542,10 @@ def map_misp_attribute_to_zeek( zeekItem = defaultdict(lambda: '-') if source is not None and len(source) > 0: - zeekItem[ZEEK_INTEL_META_SOURCE] = ','.join([x.replace(',', '\\x2c') for x in source]) + zeekItem[ZEEK_INTEL_META_SOURCE] = '\\x7c'.join([x.replace(',', '\\x2c') for x in source]) if description is not None: zeekItem[ZEEK_INTEL_META_DESC] = description + zeekItem[ZEEK_INTEL_CIF_DESCRIPTION] = zeekItem[ZEEK_INTEL_META_DESC] if url is not None: zeekItem[ZEEK_INTEL_META_URL] = url zeekItem[ZEEK_INTEL_INDICATOR] = attribute_value @@ -427,10 +557,10 @@ def map_misp_attribute_to_zeek( else: zeekItem[ZEEK_INTEL_CIF_TAGS] = attribute.category.replace(',', '\\x2c') if confidence is not None: - zeekItem[ZEEK_INTEL_CIF_CONFIDENCE] = str(confidence) + zeekItem[ZEEK_INTEL_CIF_CONFIDENCE] = str(round(confidence / 10)) results.append(zeekItem) - if logger is not None: + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): logger.debug(zeekItem) return results @@ -482,15 +612,39 @@ def PrintHeader(self): print('\t'.join(['#fields'] + self.fields), file=self.outFile) self.printedHeader = True + def ProcessMandiant(self, indicator, skip_attr_map={}): + result = False + try: + if isinstance(indicator, mandiant_threatintel.APIResponse): + # map indicator object to Zeek value(s) + if vals := map_mandiant_indicator_to_zeek( + indicator=indicator, skip_attr_map=skip_attr_map, logger=self.logger + ): + for val in vals: + self.PrintHeader() + with self.lock: + # print the intelligence item fields according to the columns in 'fields' + print('\t'.join([val[key] for key in self.fields]), file=self.outFile) + if not result: + result = True + + except Exception as e: + if self.logger is not None: + self.logger.warning( + f"{type(e).__name__} for {indicator.id if hasattr(indicator, 'id') else 'indicator'}: {e}" + ) + return result + def ProcessSTIX( self, toParse, + version=None, source: Union[Tuple[str], None] = None, ): result = False try: # parse the STIX and process all "Indicator" objects - for obj in STIXParse(toParse, allow_custom=True).objects: + for obj in STIXParse(toParse, allow_custom=True, version=version).objects: if type(obj).__name__ == "Indicator": if not result: result = True @@ -611,6 +765,354 @@ def ProcessMISP( return result +def UpdateFromMISP( + connInfo, + since, + nowTime, + sslVerify, + zeekPrinter, + logger, + successCount, + workerId, +): + with requests.Session() as mispSession: + mispSession.headers.update({'Accept': 'application/json;q=1.0,text/plain;q=0.9,text/html;q=0.9'}) + if mispAuthKey := connInfo.get('auth_key', None): + mispSession.headers.update({'Authorization': mispAuthKey}) + + mispUrl = connInfo.get('url', None) + + # download the URL and parse as JSON to figure out what it is. it could be: + # - a manifest JSON (https://www.circl.lu/doc/misp/feed-osint/manifest.json) + # - a directory listing *containing* a manifest.json (https://www.circl.lu/doc/misp/feed-osint/) + # - a directory listing of misc. JSON files without a manifest.json + # - an array of Attributes returned for a request via the MISP Automation API to an /attributes endpoint + # - an array of Events returned for a request via the MISP Automation API to an /events endpoint + mispResponse = mispSession.get( + mispUrl, + allow_redirects=True, + verify=sslVerify, + ) + mispResponse.raise_for_status() + if mispJson := LoadStrIfJson(mispResponse.content): + # the contents are JSON. determine if this is: + # - a single Event + # - an array of Events + # - an array of Attributes + # - a manifest + + if isinstance(mispJson, dict) and (len(mispJson.keys()) == 1) and ('Event' in mispJson): + # this is a single MISP Event, process it + if zeekPrinter.ProcessMISP( + mispJson, + url=mispUrl, + ): + successCount.increment() + + elif isinstance(mispJson, list) and (len(mispJson) > 0): + # are these Attributes or Events? + if isinstance(mispJson[0], dict) and ('id' in mispJson[0]) and ('type' in mispJson[0]): + controllerType = 'attributes' + resultKey = 'Attribute' + pageSize = MISP_PAGE_SIZE_ATTRIBUTES + elif isinstance(mispJson[0], dict) and ('info' in mispJson[0]): + controllerType = 'events' + resultKey = 'Event' + pageSize = MISP_PAGE_SIZE_EVENTS + else: + controllerType = None + resultKey = None + pageSize = None + + if controllerType: + # this is an array of either Attributes or Events. + # rather than handling it via additional calls with request, + # let's use the MISP API to do the searching/pulling + # (yeah, we're duplicating the effort of pulling the + # first page, but meh, who cares?) + if mispObject := PyMISP( + mispUrl, + mispAuthKey, + sslVerify, + debug=logger and (LOGGING_DEBUG >= logger.root.level), + ): + # search, looping over the pages pageSize at a time + mispPage = 0 + while True: + mispPage += 1 + resultCount = 0 + mispResults = mispObject.search( + controller=controllerType, + return_format='json', + limit=pageSize, + page=mispPage, + type_attribute=list(MISP_ZEEK_INTEL_TYPE_MAP.keys()), + timestamp=since, + ) + if mispResults and isinstance(mispResults, dict) and (resultKey in mispResults): + # Attributes results + resultCount = len(mispResults[resultKey]) + for item in mispResults[resultKey]: + try: + if zeekPrinter.ProcessMISP( + item, + url=mispUrl, + ): + successCount.increment() + except Exception as e: + if logger is not None: + logger.warning( + f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}" + ) + + elif mispResults and isinstance(mispResults, list): + # Events results + resultCount = len(mispResults) + for item in mispResults: + if item and isinstance(item, dict) and (resultKey in item): + try: + if zeekPrinter.ProcessMISP( + item[resultKey], + url=mispUrl, + ): + successCount.increment() + except Exception as e: + if logger is not None: + logger.warning( + f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}" + ) + + else: + # error or unrecognized results, set this to short circuit + resultCount = 0 + + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): + logger.debug(f"[{workerId}]: MISP search page {mispPage} returned {resultCount}") + if not mispResults or (resultCount < pageSize): + break + + else: + # not an Event or an Attribute? what the heck are we even doing? + raise Exception(f"Unknown MISP object '{json.dumps(mispJson)}'") + + elif isinstance(mispJson, dict): + # this is a manifest, loop over, retrieve and process the MISP events it references + for uri in mispJson: + try: + newUrl = urljoin(mispUrl, f'{uri}.json') + eventTime = ( + datetime.utcfromtimestamp(int(mispJson[uri]['timestamp'])).astimezone(UTCTimeZone) + if 'timestamp' in mispJson[uri] + else defaultNow + ) + if (since is None) or (eventTime >= since): + mispObjectReponse = mispSession.get( + newUrl, + allow_redirects=True, + verify=sslVerify, + ) + mispObjectReponse.raise_for_status() + if zeekPrinter.ProcessMISP( + mispObjectReponse.json(), + url=newUrl, + ): + successCount.increment() + except Exception as e: + if logger is not None: + logger.warning(f"[{workerId}]: {type(e).__name__} for MISP object at '{newUrl}': {e}") + + else: + raise Exception(f"Unknown MISP format '{type(mispJson)}'") + + else: + # the contents are NOT JSON, it's probably an HTML-formatted directory listing + + # retrieve the links listed (non-recursive, all .json files in this directory) + paths = get_url_paths_from_response(mispResponse.text, parent_url=mispUrl, ext='.json') + + # see if manifest.json exists in this directory + manifestPaths = [x for x in paths if x.endswith('/manifest.json')] + if len(manifestPaths) > 0: + # the manifest.json exists! + # retrieve it, then loop over it and retrieve and process the MISP events it references + for url in manifestPaths: + try: + mispManifestResponse = mispSession.get( + url, + allow_redirects=True, + verify=sslVerify, + ) + mispManifestResponse.raise_for_status() + mispManifest = mispManifestResponse.json() + for uri in mispManifest: + try: + eventTime = ( + datetime.utcfromtimestamp(int(mispManifest[uri]['timestamp'])).astimezone( + UTCTimeZone + ) + if 'timestamp' in mispManifest[uri] + else defaultNow + ) + if (since is None) or (eventTime >= since): + newUrl = f'{mispUrl.strip("/")}/{uri}.json' + mispObjectReponse = mispSession.get( + newUrl, + allow_redirects=True, + verify=sslVerify, + ) + mispObjectReponse.raise_for_status() + if zeekPrinter.ProcessMISP( + mispObjectReponse.json(), + url=newUrl, + ): + successCount.increment() + except Exception as e: + if logger is not None: + logger.warning( + f"[{workerId}]: {type(e).__name__} for MISP object at '{mispUrl}/{uri}.json': {e}" + ) + except Exception as e: + if logger is not None: + logger.warning(f"[{workerId}]: {type(e).__name__} for manifest at '{url}': {e}") + + else: + # the manifest.json does not exist! + # just loop over, retrieve and process the .json files in this directory + for url in paths: + try: + mispObjectReponse = mispSession.get( + url, + allow_redirects=True, + verify=sslVerify, + ) + mispObjectReponse.raise_for_status() + if zeekPrinter.ProcessMISP( + mispObjectReponse.json(), + url=url, + ): + successCount.increment() + except Exception as e: + if logger is not None: + logger.warning(f"[{workerId}]: {type(e).__name__} for MISP object at '{url}': {e}") + + +def UpdateFromTAXII( + connInfo, + since, + nowTime, + sslVerify, + zeekPrinter, + logger, + successCount, + workerId, +): + # connect to the server with the appropriate API for the TAXII version + taxiiUrl = connInfo.get('url', None) + taxiiCollection = connInfo.get('collection', None) + taxiiUsername = connInfo.get('username', None) + taxiiPassword = connInfo.get('password', None) + taxiiVersion = str(connInfo.get('version', None)) + if taxiiVersion == '2.0': + TaxiiServerClass = TaxiiServer_v20 + TaxiiCollectionClass = TaxiiCollection_v20 + TaxiiAsPagesClass = TaxiiAsPages_v20 + elif taxiiVersion == '2.1': + TaxiiServerClass = TaxiiServer_v21 + TaxiiCollectionClass = TaxiiCollection_v21 + TaxiiAsPagesClass = TaxiiAsPages_v21 + else: + raise Exception(f"Unsupported TAXII version '{taxiiVersion}'") + + server = TaxiiServerClass(taxiiUrl, user=taxiiUsername, password=taxiiPassword, verify=sslVerify) + + # collect the collection URL(s) for the given collection name + collectionUrls = {} + for api_root in server.api_roots: + for collection in api_root.collections: + if (taxiiCollection == '*') or (collection.title.lower() == taxiiCollection.lower()): + collectionUrls[collection.title] = { + 'id': collection.id, + 'url': collection.url, + } + + # connect to and retrieve indicator STIX objects from the collection URL(s) + for title, info in collectionUrls.items(): + collection = TaxiiCollectionClass( + info['url'], + user=taxiiUsername, + password=taxiiPassword, + verify=sslVerify, + ) + try: + # loop over paginated results + for envelope in TaxiiAsPagesClass( + collection.get_objects, + per_request=TAXII_PAGE_SIZE, + **TAXII_INDICATOR_FILTER, + ): + if zeekPrinter.ProcessSTIX( + envelope, + version=taxiiVersion, + source=[':'.join([x for x in [server.title, title] if x is not None])], + ): + successCount.increment() + + except Exception as e: + if logger is not None: + logger.warning(f"[{workerId}]: {type(e).__name__} for object of collection '{title}': {e}") + + +def UpdateFromMandiant( + connInfo, + since, + nowTime, + sslVerify, + zeekPrinter, + logger, + successCount, + workerId, +): + if mati_client := mandiant_threatintel.ThreatIntelClient( + api_key=connInfo.get('api_key', None), + secret_key=connInfo.get('secret_key', None), + bearer_token=connInfo.get('bearer_token', None), + api_base_url=connInfo.get('api_base_url', mandiant_threatintel.API_BASE_URL), + client_name=connInfo.get('client_name', mandiant_threatintel.CLIENT_APP_NAME), + ): + skip_attr_map = defaultdict(lambda: False) + skip_attr_map['campaigns'] = not bool(connInfo.get('include_campaigns', MANDIANT_INCLUDE_CAMPAIGNS_DEFAULT)) + skip_attr_map['category'] = not bool(connInfo.get('include_category', MANDIANT_INCLUDE_CATEGORY_DEFAULT)) + skip_attr_map['misp'] = not bool(connInfo.get('include_misp', MANDIANT_INCLUDE_MISP_DEFAULT)) + skip_attr_map['reports'] = not bool(connInfo.get('include_reports', MANDIANT_INCLUDE_REPORTS_DEFAULT)) + skip_attr_map['threat_rating'] = not bool( + connInfo.get('include_threat_rating', MANDIANT_INCLUDE_THREAT_RATING_DEFAULT) + ) + skip_attr_map['attributed_associations'] = True + for indicator in mati_client.Indicators.get_list( + start_epoch=since if since else nowTime - relativedelta(hours=24), + end_epoch=nowTime, + page_size=connInfo.get('page_size', MANDIANT_PAGE_SIZE_DEFAULT), + minimum_mscore=connInfo.get('minimum_mscore', MANDIANT_MINIMUM_MSCORE_DEFAULT), + exclude_osint=connInfo.get('exclude_osint', MANDIANT_EXCLUDE_OSINT_DEFAULT), + include_campaigns=not skip_attr_map['campaigns'], + include_reports=not skip_attr_map['reports'], + include_threat_rating=not skip_attr_map['threat_rating'], + include_misp=not skip_attr_map['misp'], + include_category=skip_attr_map['category'], + ): + try: + if zeekPrinter.ProcessMandiant(indicator, skip_attr_map=skip_attr_map): + successCount.increment() + except Exception as e: + if logger is not None: + logger.warning( + f"[{workerId}]: {type(e).__name__} for Mandiant indicator {indicator.id if isinstance(indicator, mandiant_threatintel.APIResponse) else ''}: {e}" + ) + + else: + raise Exception("Could not connect to Mandiant threat intelligence service") + + def ProcessThreatInputWorker(threatInputWorkerArgs): inputQueue, zeekPrinter, since, sslVerify, defaultNow, workerThreadCount, successCount, logger = ( threatInputWorkerArgs[0], @@ -624,7 +1126,7 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): ) with workerThreadCount as workerId: - if logger is not None: + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): logger.debug(f"[{workerId}]: started") # the queue was fully populated before we started, so we can run until there are no more elements @@ -635,7 +1137,11 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): sleep(1) else: try: - with open(inarg) if ((inarg is not None) and os.path.isfile(inarg)) else nullcontext() as infile: + with ( + open(inarg) + if (isinstance(inarg, (str, bytes, os.PathLike, int)) and os.path.isfile(inarg)) + else nullcontext() + ) as infile: if infile: ################################################################################## # JSON FILE (STIX or MISP) @@ -664,270 +1170,78 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): else: raise Exception(f"Could not parse JSON in '{inarg}'") - elif inarg.lower().startswith('misp'): + elif isinstance(inarg, dict): + ################################################################################## + # Connection parameters specified in dict (e.g., Mandiant Threat Intel) from a YAML file + if ('type' in inarg) and (threatFeedType := str(inarg['type'])): + if threatFeedType.lower() == 'misp': + UpdateFromMISP( + inarg, + since, + defaultNow, + sslVerify, + zeekPrinter, + logger, + successCount, + workerId, + ) + elif threatFeedType.lower() == 'taxii': + UpdateFromTAXII( + inarg, + since, + defaultNow, + sslVerify, + zeekPrinter, + logger, + successCount, + workerId, + ) + elif threatFeedType.lower() == 'mandiant': + UpdateFromMandiant( + inarg, + since, + defaultNow, + sslVerify, + zeekPrinter, + logger, + successCount, + workerId, + ) + else: + raise Exception(f"Could not handle identify threat feed type '{threatFeedType}'") + else: + raise Exception(f"Could not identify threat feed type in '{inarg}'") + + elif isinstance(inarg, str) and inarg.lower().startswith('misp'): ################################################################################## # MISP URL + # this is a MISP URL, connect and retrieve MISP indicators from it - # this is a MISP URL, connect and retrieve STIX indicators from it + mispConnInfoDict = defaultdict(lambda: None) + mispConnInfoDict['type'] = 'misp' # misp|misp_url|auth_key - - mispConnInfo = [base64_decode_if_prefixed(x) for x in inarg.split('|')[1::]] - mispUrl, mispAuthKey = ( - None, - None, + mispConnInfoParts = [base64_decode_if_prefixed(x) for x in inarg.split('|')[1::]] + mispConnInfoDict['url'] = mispConnInfoParts[0] + if len(mispConnInfoParts) >= 2: + mispConnInfoDict['auth_key'] = mispConnInfoParts[1] + UpdateFromMISP( + mispConnInfoDict, + since, + defaultNow, + sslVerify, + zeekPrinter, + logger, + successCount, + workerId, ) - mispUrl = mispConnInfo[0] - if len(mispConnInfo) >= 2: - mispAuthKey = mispConnInfo[1] - with requests.Session() as mispSession: - mispSession.headers.update( - {'Accept': 'application/json;q=1.0,text/plain;q=0.9,text/html;q=0.9'} - ) - if mispAuthKey is not None: - mispSession.headers.update({'Authorization': mispAuthKey}) - - # download the URL and parse as JSON to figure out what it is. it could be: - # - a manifest JSON (https://www.circl.lu/doc/misp/feed-osint/manifest.json) - # - a directory listing *containing* a manifest.json (https://www.circl.lu/doc/misp/feed-osint/) - # - a directory listing of misc. JSON files without a manifest.json - # - an array of Attributes returned for a request via the MISP Automation API to an /attributes endpoint - # - an array of Events returned for a request via the MISP Automation API to an /events endpoint - mispResponse = mispSession.get( - mispUrl, - allow_redirects=True, - verify=sslVerify, - ) - mispResponse.raise_for_status() - if mispJson := LoadStrIfJson(mispResponse.content): - # the contents are JSON. determine if this is: - # - a single Event - # - an array of Events - # - an array of Attributes - # - a manifest - - if ( - isinstance(mispJson, dict) - and (len(mispJson.keys()) == 1) - and ('Event' in mispJson) - ): - # this is a single MISP Event, process it - if zeekPrinter.ProcessMISP( - mispJson, - url=mispUrl, - ): - successCount.increment() - - elif isinstance(mispJson, list) and (len(mispJson) > 0): - # are these Attributes or Events? - if ( - isinstance(mispJson[0], dict) - and ('id' in mispJson[0]) - and ('type' in mispJson[0]) - ): - controllerType = 'attributes' - resultKey = 'Attribute' - pageSize = MISP_PAGE_SIZE_ATTRIBUTES - elif isinstance(mispJson[0], dict) and ('info' in mispJson[0]): - controllerType = 'events' - resultKey = 'Event' - pageSize = MISP_PAGE_SIZE_EVENTS - else: - controllerType = None - resultKey = None - pageSize = None - - if controllerType: - # this is an array of either Attributes or Events. - # rather than handling it via additional calls with request, - # let's use the MISP API to do the searching/pulling - # (yeah, we're duplicating the effort of pulling the - # first page, but meh, who cares?) - if mispObject := PyMISP( - mispUrl, - mispAuthKey, - sslVerify, - debug=logger and (LOGGING_DEBUG >= logger.root.level), - ): - # search, looping over the pages pageSize at a time - mispPage = 0 - while True: - mispPage += 1 - resultCount = 0 - mispResults = mispObject.search( - controller=controllerType, - return_format='json', - limit=pageSize, - page=mispPage, - type_attribute=list(MISP_ZEEK_INTEL_TYPE_MAP.keys()), - timestamp=since, - ) - if ( - mispResults - and isinstance(mispResults, dict) - and (resultKey in mispResults) - ): - # Attributes results - resultCount = len(mispResults[resultKey]) - for item in mispResults[resultKey]: - try: - if zeekPrinter.ProcessMISP( - item, - url=mispUrl, - ): - successCount.increment() - except Exception as e: - if logger is not None: - logger.warning( - f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}" - ) - - elif mispResults and isinstance(mispResults, list): - # Events results - resultCount = len(mispResults) - for item in mispResults: - if item and isinstance(item, dict) and (resultKey in item): - try: - if zeekPrinter.ProcessMISP( - item[resultKey], - url=mispUrl, - ): - successCount.increment() - except Exception as e: - if logger is not None: - logger.warning( - f"[{workerId}]: {type(e).__name__} for MISP {resultKey}: {e}" - ) - - else: - # error or unrecognized results, set this to short circuit - resultCount = 0 - - if logger is not None: - logger.debug( - f"[{workerId}]: MISP search page {mispPage} returned {resultCount}" - ) - if not mispResults or (resultCount < pageSize): - break - - else: - # not an Event or an Attribute? what the heck are we even doing? - raise Exception(f"Unknown MISP object '{json.dumps(mispJson)}'") - - elif isinstance(mispJson, dict): - # this is a manifest, loop over, retrieve and process the MISP events it references - for uri in mispJson: - try: - newUrl = urljoin(mispUrl, f'{uri}.json') - eventTime = ( - datetime.utcfromtimestamp( - int(mispJson[uri]['timestamp']) - ).astimezone(UTCTimeZone) - if 'timestamp' in mispJson[uri] - else defaultNow - ) - if (since is None) or (eventTime >= since): - mispObjectReponse = mispSession.get( - newUrl, - allow_redirects=True, - verify=sslVerify, - ) - mispObjectReponse.raise_for_status() - if zeekPrinter.ProcessMISP( - mispObjectReponse.json(), - url=newUrl, - ): - successCount.increment() - except Exception as e: - if logger is not None: - logger.warning( - f"[{workerId}]: {type(e).__name__} for MISP object at '{newUrl}': {e}" - ) - - else: - raise Exception(f"Unknown MISP format '{type(mispJson)}'") - - else: - # the contents are NOT JSON, it's probably an HTML-formatted directory listing - - # retrieve the links listed (non-recursive, all .json files in this directory) - paths = get_url_paths_from_response( - mispResponse.text, parent_url=mispUrl, ext='.json' - ) - - # see if manifest.json exists in this directory - manifestPaths = [x for x in paths if x.endswith('/manifest.json')] - if len(manifestPaths) > 0: - # the manifest.json exists! - # retrieve it, then loop over it and retrieve and process the MISP events it references - for url in manifestPaths: - try: - mispManifestResponse = mispSession.get( - url, - allow_redirects=True, - verify=sslVerify, - ) - mispManifestResponse.raise_for_status() - mispManifest = mispManifestResponse.json() - for uri in mispManifest: - try: - eventTime = ( - datetime.utcfromtimestamp( - int(mispManifest[uri]['timestamp']) - ).astimezone(UTCTimeZone) - if 'timestamp' in mispManifest[uri] - else defaultNow - ) - if (since is None) or (eventTime >= since): - newUrl = f'{mispUrl.strip("/")}/{uri}.json' - mispObjectReponse = mispSession.get( - newUrl, - allow_redirects=True, - verify=sslVerify, - ) - mispObjectReponse.raise_for_status() - if zeekPrinter.ProcessMISP( - mispObjectReponse.json(), - url=newUrl, - ): - successCount.increment() - except Exception as e: - if logger is not None: - logger.warning( - f"[{workerId}]: {type(e).__name__} for MISP object at '{mispUrl}/{uri}.json': {e}" - ) - except Exception as e: - if logger is not None: - logger.warning( - f"[{workerId}]: {type(e).__name__} for manifest at '{url}': {e}" - ) - - else: - # the manifest.json does not exist! - # just loop over, retrieve and process the .json files in this directory - for url in paths: - try: - mispObjectReponse = mispSession.get( - url, - allow_redirects=True, - verify=sslVerify, - ) - mispObjectReponse.raise_for_status() - if zeekPrinter.ProcessMISP( - mispObjectReponse.json(), - url=url, - ): - successCount.increment() - except Exception as e: - if logger is not None: - logger.warning( - f"[{workerId}]: {type(e).__name__} for MISP object at '{url}': {e}" - ) - - elif inarg.lower().startswith('taxii'): + elif isinstance(inarg, str) and inarg.lower().startswith('taxii'): ################################################################################## # TAXI (STIX) URL + taxiiConnInfoDict = defaultdict(lambda: None) + taxiiConnInfoDict['type'] = 'taxii' + # this is a TAXII URL, connect and retrieve STIX indicators from it # taxii|2.0|discovery_url|collection_name|username|password # @@ -936,87 +1250,32 @@ def ProcessThreatInputWorker(threatInputWorkerArgs): # - "taxii|2.0|https://limo.anomali.com/api/v1/taxii2/taxii/|CyberCrime|guest|guest" # # collection_name can be specified as * to retrieve all collections (careful!) - taxiiConnInfo = [base64_decode_if_prefixed(x) for x in inarg.split('|')[1::]] - taxiiVersion, taxiiDisoveryURL, taxiiCollectionName, taxiiUsername, taxiiPassword = ( - None, - None, - None, - None, - None, - ) if len(taxiiConnInfo) >= 3: - taxiiVersion, taxiiDisoveryURL, taxiiCollectionName = taxiiConnInfo[0:3] + ( + taxiiConnInfoDict['version'], + taxiiConnInfoDict['url'], + taxiiConnInfoDict['collection'], + ) = taxiiConnInfo[0:3] if len(taxiiConnInfo) >= 4: - taxiiUsername = taxiiConnInfo[3] + taxiiConnInfoDict['username'] = taxiiConnInfo[3] if len(taxiiConnInfo) >= 5: - taxiiPassword = taxiiConnInfo[4] - - # connect to the server with the appropriate API for the TAXII version - if taxiiVersion == '2.0': - server = TaxiiServer_v20( - taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword, verify=sslVerify - ) - elif taxiiVersion == '2.1': - server = TaxiiServer_v21( - taxiiDisoveryURL, user=taxiiUsername, password=taxiiPassword, verify=sslVerify - ) - else: - raise Exception(f"Unsupported TAXII version '{taxiiVersion}'") - - # collect the collection URL(s) for the given collection name - collectionUrls = {} - for api_root in server.api_roots: - for collection in api_root.collections: - if (taxiiCollectionName == '*') or ( - collection.title.lower() == taxiiCollectionName.lower() - ): - collectionUrls[collection.title] = { - 'id': collection.id, - 'url': collection.url, - } - - # connect to and retrieve indicator STIX objects from the collection URL(s) - for title, info in collectionUrls.items(): - collection = ( - TaxiiCollection_v21( - info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify - ) - if taxiiVersion == '2.1' - else TaxiiCollection_v20( - info['url'], user=taxiiUsername, password=taxiiPassword, verify=sslVerify - ) - ) - try: - # loop over paginated results - for envelope in ( - TaxiiAsPages_v21( - collection.get_objects, - per_request=TAXII_PAGE_SIZE, - **TAXII_INDICATOR_FILTER, - ) - if taxiiVersion == '2.1' - else TaxiiAsPages_v20( - collection.get_objects, - per_request=TAXII_PAGE_SIZE, - **TAXII_INDICATOR_FILTER, - ) - ): - if zeekPrinter.ProcessSTIX( - envelope, - source=[':'.join([x for x in [server.title, title] if x is not None])], - ): - successCount.increment() - - except Exception as e: - if logger is not None: - logger.warning( - f"[{workerId}]: {type(e).__name__} for object of collection '{title}': {e}" - ) + taxiiConnInfoDict['password'] = taxiiConnInfo[4] + + UpdateFromTAXII( + taxiiConnInfoDict, + since, + defaultNow, + sslVerify, + zeekPrinter, + logger, + successCount, + workerId, + ) except Exception as e: if logger is not None: logger.warning(f"[{workerId}]: {type(e).__name__} for '{inarg}': {e}") - if logger is not None: + if (logger is not None) and (LOGGING_DEBUG >= logger.root.level): logger.debug(f"[{workerId}]: finished") diff --git a/shared/bin/zeekdeploy.sh b/shared/bin/zeekdeploy.sh index c931e592c..2500f5167 100755 --- a/shared/bin/zeekdeploy.sh +++ b/shared/bin/zeekdeploy.sh @@ -155,7 +155,7 @@ fi # make sure "intel" directory exists, even if empty [[ -n "$ZEEK_INTEL_PATH" ]] && INTEL_DIR="$ZEEK_INTEL_PATH" || INTEL_DIR=/opt/sensor/sensor_ctl/zeek/intel export INTEL_DIR -mkdir -p "$INTEL_DIR"/STIX "$INTEL_DIR"/MISP +mkdir -p "$INTEL_DIR"/STIX "$INTEL_DIR"/MISP "$INTEL_DIR"/Mandiant touch "$INTEL_DIR"/__load__.zeek 2>/dev/null || true # autoconfigure load directives for intel files [[ -x "$ZEEK_INSTALL_PATH"/bin/zeek_intel_setup.sh ]] && \ diff --git a/zeek/config/known-routers.zeek b/zeek/config/known-routers.zeek new file mode 100644 index 000000000..b77d97b00 --- /dev/null +++ b/zeek/config/known-routers.zeek @@ -0,0 +1,173 @@ +##! This script logs routers, devices that Zeek determines have +##! sent IPv4 packets with TTL 255 or IPv6 packets with HLIM 255, +##! and logs the address once per day (by default). It is modeled +##! pretty heavily after known-hosts.zeek + +@load base/utils/directions-and-hosts +@load base/frameworks/cluster + +module Known; + +export { + ## The known-routers logging stream identifier. + redef enum Log::ID += { ROUTERS_LOG }; + + ## A default logging policy hook for the stream. + global log_policy_routers: Log::PolicyHook; + + ## The record type which contains the column fields of the known-routers log. + type RouterInfo: record { + ## The timestamp at which the router was detected. + ts: time &log; + ## The originating IPv4 or IPv6 address of the detected packet with the 255 TTL/HLIM value. + orig_h: addr &log; + ## The originating MAC address of the detected packet with the 255 TTL/HLIM value. + orig_l2_addr: string &log &optional; + ## When IPv4, the TTL value. + ttl: count &log &optional; + ## When IPv6, the HLIM value. + hlim: count &log &optional; + }; + + ## Toggles between different implementations of this script. + ## When true, use a Broker data store, else use a regular Zeek set + ## with keys uniformly distributed over proxy nodes in cluster + ## operation. + const use_router_store = F &redef; + + ## The router hosts whose existence should be logged and tracked. + ## See :zeek:type:`Host` for possible choices. + option router_tracking = LOCAL_HOSTS; + + ## Holds the set of all known routers. Keys in the store are addresses + ## and their associated value will always be the "true" boolean. + global router_store: Cluster::StoreInfo; + + ## The Broker topic name to use for :zeek:see:`Known::router_store`. + const router_store_name = "zeek/known/routers" &redef; + + ## The expiry interval of new entries in :zeek:see:`Known::router_store`. + ## This also changes the interval at which routers get logged. + const router_store_expiry = 1day &redef; + + ## The timeout interval to use for operations against + ## :zeek:see:`Known::router_store`. + option router_store_timeout = 15sec; + + ## The set of all known addresses to store for preventing duplicate + ## logging of addresses. It can also be used from other scripts to + ## inspect if an address has been seen in use. + ## Maintain the list of known routers for 24 hours so that the existence + ## of each individual address is logged each day. + ## + ## In cluster operation, this set is distributed uniformly across + ## proxy nodes. + global routers: set[addr] &create_expire=1day &redef; + + ## An event that can be handled to access the :zeek:type:`Known::RouterInfo` + ## record as it is sent on to the logging framework. + global log_known_routers: event(rec: RouterInfo); +} + +event zeek_init() { + if ( ! Known::use_router_store ) + return; + Known::router_store = Cluster::create_store(Known::router_store_name); +} + +event Known::router_found(info: RouterInfo) { + if ( ! Known::use_router_store ) + return; + + when [info] ( local r = Broker::put_unique(Known::router_store$store, + info$orig_h, + T, + Known::router_store_expiry) ) { + if ( r$status == Broker::SUCCESS ) { + if ( r$result as bool ) + Log::write(Known::ROUTERS_LOG, info); + } else + Reporter::error(fmt("%s: data store put_unique failure", + Known::router_store_name)); + } + + timeout Known::router_store_timeout { + # Can't really tell if master store ended up inserting a key. + Log::write(Known::ROUTERS_LOG, info); + } +} + +event known_router_add(info: RouterInfo) { + if ( use_router_store ) + return; + + if ( info$orig_h in Known::routers ) + return; + + add Known::routers[info$orig_h]; + + @if ( ! Cluster::is_enabled() || Cluster::local_node_type() == Cluster::PROXY ) + Log::write(Known::ROUTERS_LOG, info); + @endif +} + +event Cluster::node_up(name: string, id: string) { + if ( use_router_store ) + return; + + if ( Cluster::local_node_type() != Cluster::WORKER ) + return; + + # Drop local suppression cache on workers to force HRW key repartitioning. + clear_table(Known::routers); +} + +event Cluster::node_down(name: string, id: string) { + if ( use_router_store ) + return; + + if ( Cluster::local_node_type() != Cluster::WORKER ) + return; + + # Drop local suppression cache on workers to force HRW key repartitioning. + clear_table(Known::routers); +} + +event Known::router_found(info: RouterInfo) { + if ( use_router_store ) + return; + + if ( info$orig_h in Known::routers ) + return; + + Cluster::publish_hrw(Cluster::proxy_pool, info$orig_h, known_router_add, info); + event known_router_add(info); +} + +event zeek_init() &priority=5 { + Log::create_stream(Known::ROUTERS_LOG, [$columns=RouterInfo, + $ev=log_known_routers, + $path="known_routers", + $policy=log_policy_routers]); +} + +event new_connection(c: connection) &priority=5 { + local p: raw_pkt_hdr = get_current_packet_header(); + + if ( ( ( p?$ip && ( p$ip$ttl == 255 ) ) || ( p?$ip6 && ( p$ip6$hlim == 255 ) ) ) && ( addr_matches_host(c$id$orig_h, router_tracking) ) ) { + + local ttl : count = 0; + local hlim : count = 0; + local mac : string = ""; + if ( p?$ip ) ttl = p$ip$ttl; + else if ( p?$ip6 ) hlim = p$ip6$hlim; + if ( c?$orig && c$orig?$l2_addr ) mac = c$orig$l2_addr; + + event Known::router_found([$ts=network_time(), + $orig_h=c$id$orig_h, + $ttl=ttl, + $hlim=hlim, + $orig_l2_addr=mac]); + + } +} \ No newline at end of file diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek index a2fef6e6c..ac8167e1d 100644 --- a/zeek/config/local.zeek +++ b/zeek/config/local.zeek @@ -12,6 +12,7 @@ global disable_log_passwords = (getenv("ZEEK_DISABLE_LOG_PASSWORDS") == true_reg global disable_ssl_validate_certs = (getenv("ZEEK_DISABLE_SSL_VALIDATE_CERTS") == true_regex) ? T : F; global disable_track_all_assets = (getenv("ZEEK_DISABLE_TRACK_ALL_ASSETS") == true_regex) ? T : F; global disable_best_guess_ics = (getenv("ZEEK_DISABLE_BEST_GUESS_ICS") == true_regex) ? T : F; +global disable_detect_routers = (getenv("ZEEK_DISABLE_DETECT_ROUTERS") == true_regex) ? T : F; global synchrophasor_detailed = (getenv("ZEEK_SYNCHROPHASOR_DETAILED") == true_regex) ? T : F; global synchrophasor_ports_str = getenv("ZEEK_SYNCHROPHASOR_PORTS"); global genisys_ports_str = getenv("ZEEK_GENISYS_PORTS"); @@ -44,6 +45,10 @@ global disable_ics_profinet_io_cm = (getenv("ZEEK_DISABLE_ICS_PROFINET_IO_CM") = global disable_ics_s7comm = (getenv("ZEEK_DISABLE_ICS_S7COMM") == true_regex) ? T : F; global disable_ics_synchrophasor = (getenv("ZEEK_DISABLE_ICS_SYNCHROPHASOR") == true_regex) ? T : F; +global zeek_kafka_enabled = (getenv("ZEEK_KAFKA_ENABLED") == true_regex) ? T : F; +global zeek_kafka_brokers = getenv("ZEEK_KAFKA_BROKERS"); +global zeek_kafka_topic = getenv("ZEEK_KAFKA_TOPIC"); + redef Broker::default_listen_address = "127.0.0.1"; redef ignore_checksums = T; @@ -105,6 +110,9 @@ global json_format = (getenv("ZEEK_JSON") == true_regex) ? T : F; @if (!disable_best_guess_ics) @load ./guess.zeek @endif +@if (!disable_detect_routers) + @load ./known-routers.zeek +@endif @load packages @if (!disable_ics_all) @@ -344,3 +352,13 @@ hook PacketAnalyzer::ECAT::log_policy_ecat_arp( filter: Log::Filter) { break; } + +@if (zeek_kafka_enabled) + @load packages/zeek-kafka + redef Kafka::send_all_active_logs = T; + redef Kafka::topic_name = zeek_kafka_topic; + redef Kafka::tag_json = T; + redef Kafka::kafka_conf = table( + ["metadata.broker.list"] = zeek_kafka_brokers +); +@endif diff --git a/zeek/intel/Mandiant/.gitignore b/zeek/intel/Mandiant/.gitignore new file mode 100644 index 000000000..a5baada18 --- /dev/null +++ b/zeek/intel/Mandiant/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore + diff --git a/zeek/scripts/container_health.sh b/zeek/scripts/container_health.sh index 2fa834f4c..75464199f 100755 --- a/zeek/scripts/container_health.sh +++ b/zeek/scripts/container_health.sh @@ -5,7 +5,7 @@ if [[ "${ZEEK_LIVE_CAPTURE:-false}" == "true" ]]; then supervisorctl status live-zeek >/dev/null 2>&1 else - if [[ "${ZEEK_INTEL_REFRESH_ON_ENTRYPOINT:-false}" == "true" ]]; then + if [[ "${ZEEK_INTEL_REFRESH_ON_STARTUP:-false}" == "true" ]]; then ( ps a 2>/dev/null | grep -q '[z]eek_intel_from_threat_feed.py' ) || supervisorctl status pcap-zeek >/dev/null 2>&1 else supervisorctl status pcap-zeek >/dev/null 2>&1 diff --git a/zeek/scripts/docker_entrypoint.sh b/zeek/scripts/docker_entrypoint.sh index 34aa5f82b..88be6bdd1 100755 --- a/zeek/scripts/docker_entrypoint.sh +++ b/zeek/scripts/docker_entrypoint.sh @@ -6,16 +6,5 @@ ZEEK_DIR=${ZEEK_DIR:-"/opt/zeek"} setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/zeek 2>/dev/null || true setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' "${ZEEK_DIR}"/bin/capstats 2>/dev/null || true -if [[ "${ZEEK_INTEL_REFRESH_ON_ENTRYPOINT:-false}" == "true" ]] && \ - [[ -x "${ZEEK_DIR}"/bin/zeek_intel_setup.sh ]]; then - if [[ "$(id -u)" == "0" ]] && [[ -n "$PUSER" ]]; then - su -s /bin/bash -p ${PUSER} << EOF - "${ZEEK_DIR}"/bin/zeek_intel_setup.sh /bin/true -EOF - else - "${ZEEK_DIR}"/bin/zeek_intel_setup.sh /bin/true - fi -fi - # start supervisor (which will spawn pcap-zeek, cron, etc.) or whatever the default command is exec "$@" diff --git a/zeek/supervisord.conf b/zeek/supervisord.conf index ec258c16d..3c9a89b1c 100644 --- a/zeek/supervisord.conf +++ b/zeek/supervisord.conf @@ -53,6 +53,17 @@ stdout_logfile_maxbytes=0 redirect_stderr=true user=%(ENV_PUSER)s +[program:intel-initialization] +command="%(ENV_ZEEK_DIR)s"/bin/zeek_intel_setup.sh /bin/true +autostart=%(ENV_ZEEK_INTEL_REFRESH_ON_STARTUP)s +autorestart=false +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true +user=%(ENV_PUSER)s + [program:live-zeek] command=/opt/zeek/bin/zeekdeploy.sh autostart=%(ENV_ZEEK_LIVE_CAPTURE)s