From c43b48ee5a04c60f6bd87dfb7ce8794f7d9ff871 Mon Sep 17 00:00:00 2001 From: KayJay7 <31775749+KayJay7@users.noreply.github.com> Date: Tue, 26 Nov 2024 14:05:42 +0000 Subject: [PATCH 1/4] Added IPNEIGH plugin --- front/plugins/ipneigh/README.md | 22 ++ front/plugins/ipneigh/config.json | 434 ++++++++++++++++++++++++++++++ front/plugins/ipneigh/ipneigh.py | 152 +++++++++++ 3 files changed, 608 insertions(+) create mode 100755 front/plugins/ipneigh/README.md create mode 100755 front/plugins/ipneigh/config.json create mode 100755 front/plugins/ipneigh/ipneigh.py diff --git a/front/plugins/ipneigh/README.md b/front/plugins/ipneigh/README.md new file mode 100755 index 000000000..dca94ddf7 --- /dev/null +++ b/front/plugins/ipneigh/README.md @@ -0,0 +1,22 @@ +## Overview + +This plugin reads from the ARP and NDP tables using the `ip neigh` command. + +This differs from the `ARPSCAN` plugin because +* It does *not* send arp requests, it just reads the table +* It supports IPv6 +* It sends an IPv6 multicast ping to solicit IPv6 neighbour discovery + +### Quick setup guide + +To set up the plugin correctly, make sure to add in the plugin settings the name of the interfaces you want to scan. This plugin doesn't use the global `SCAN_SUBNET` setting, this is because by design it is not aware of subnets, it just looks at all the IPs reachable from an interface. + +### Usage + +- Head to **Settings** > **IP Neigh** to add the interfaces you want to scan to the `IPNEIGH_interfaces` option +- The interface list must be formatted without whitespaces and comma separated + +### Notes + +- `ARPSCAN` does a better job at discovering IPv4 devices because it explicitly sends arp requests +- IPv6 devices can \ No newline at end of file diff --git a/front/plugins/ipneigh/config.json b/front/plugins/ipneigh/config.json new file mode 100755 index 000000000..64b73ead5 --- /dev/null +++ b/front/plugins/ipneigh/config.json @@ -0,0 +1,434 @@ +{ + "code_name": "ipneigh", + "unique_prefix": "IPNEIGH", + "plugin_type": "device_scanner", + "execution_order": "Layer_0", + "enabled": true, + "data_source": "script", + "mapped_to_table": "CurrentScan", + "data_filters": [ + { + "compare_column": "Object_PrimaryID", + "compare_operator": "==", + "compare_field_id": "txtMacFilter", + "compare_js_template": "'{value}'.toString()", + "compare_use_quotes": true + } + ], + "show_ui": true, + "localized": [ + "display_name", + "description", + "icon" + ], + "display_name": [ + { + "language_code": "en_us", + "string": "IP Neigh" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "Plugin to scan the ip tables" + } + ], + "icon": [ + { + "language_code": "en_us", + "string": "" + } + ], + "params": [], + "settings": [ + { + "function": "RUN", + "events": [ + "run" + ], + "type": { + "dataType": "string", + "elements": [ + { + "elementType": "select", + "elementOptions": [], + "transformers": [] + } + ] + }, + "default_value": "disabled", + "options": [ + "disabled", + "once", + "schedule", + "always_after_scan", + "on_new_device", + "on_notification" + ], + "localized": [ + "name", + "description" + ], + "name": [ + { + "language_code": "en_us", + "string": "When to run" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "When the plugin should run. Good options are always_after_scan, on_new_device, on_notification" + } + ] + }, + { + "function": "RUN_SCHD", + "type": { + "dataType": "string", + "elements": [ + { + "elementType": "input", + "elementOptions": [], + "transformers": [] + } + ] + }, + "default_value": "*/5 * * * *", + "options": [], + "localized": [ + "name", + "description" + ], + "name": [ + { + "language_code": "en_us", + "string": "Schedule" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "Only enabled if you select schedule in the SYNC_RUN setting. Make sure you enter the schedule in the correct cron-like format (e.g. validate at crontab.guru). For example entering 0 4 * * * will run the scan after 4 am in the TIMEZONE you set above. Will be run NEXT time the time passes." + } + ] + }, + { + "function": "interfaces", + "type": { + "dataType": "string", + "elements": [ + { + "elementType": "input", + "elementOptions": [], + "transformers": [] + } + ] + }, + "maxLength": 150, + "default_value": "", + "options": [], + "localized": [ + "name", + "description" + ], + "name": [ + { + "language_code": "en_us", + "string": "Interfaces to scan" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "The plugin will scan these comma separated interfaces" + } + ] + }, + { + "function": "CMD", + "type": { + "dataType": "string", + "elements": [ + { + "elementType": "input", + "elementOptions": [ + { + "readonly": "true" + } + ], + "transformers": [] + } + ] + }, + "default_value": "python3 /app/front/plugins/ipneigh/ipneigh.py ipneigh_interfaces={IPNEIGH_interfaces}", + "options": [], + "localized": [ + "name", + "description" + ], + "name": [ + { + "language_code": "en_us", + "string": "Command" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "Command to run. This can not be changed" + } + ] + }, + { + "function": "RUN_TIMEOUT", + "type": { + "dataType": "integer", + "elements": [ + { + "elementType": "input", + "elementOptions": [ + { + "type": "number" + } + ], + "transformers": [] + } + ] + }, + "default_value": 30, + "options": [], + "localized": [ + "name", + "description" + ], + "name": [ + { + "language_code": "en_us", + "string": "Run timeout" + } + ], + "description": [ + { + "language_code": "en_us", + "string": "Maximum time in seconds to wait for the script to finish. If this time is exceeded the script is aborted." + } + ] + } + ], + "database_column_definitions": [ + { + "column": "Index", + "css_classes": "col-sm-2", + "show": true, + "type": "none", + "default_value": "", + "options": [], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "Index" + } + ] + }, + { + "column": "Object_PrimaryID", + "mapped_to_column": "cur_MAC", + "css_classes": "col-sm-2", + "show": true, + "type": "device_name_mac", + "default_value": "", + "options": [], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "MAC" + } + ] + }, + { + "column": "Object_SecondaryID", + "mapped_to_column": "cur_IP", + "css_classes": "col-sm-2", + "show": true, + "type": "device_ip", + "default_value": "", + "options": [], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "IP" + } + ] + }, + { + "column": "Watched_Value1", + "mapped_to_column": "cur_Name", + "css_classes": "col-sm-2", + "show": true, + "type": "label", + "default_value": "", + "options": [], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "Name" + } + ] + }, + { + "column": "Watched_Value2", + "mapped_to_column": "cur_Vendor", + "css_classes": "col-sm-2", + "show": true, + "type": "label", + "default_value": "", + "options": [], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "Vendor" + } + ] + }, + { + "column": "Watched_Value3", + "mapped_to_column": "cur_Type", + "css_classes": "col-sm-2", + "show": true, + "type": "label", + "default_value": "", + "options": [], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "Device Type" + } + ] + }, + { + "column": "Watched_Value4", + "css_classes": "col-sm-2", + "show": false, + "type": "label", + "default_value": "", + "options": [], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "N/A" + } + ] + }, + { + "column": "Dummy", + "mapped_to_column": "cur_ScanMethod", + "mapped_to_column_data": { + "value": "ip neighbor" + }, + "css_classes": "col-sm-2", + "show": true, + "type": "label", + "default_value": "", + "options": [], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "Scan method" + } + ] + }, + { + "column": "DateTimeCreated", + "css_classes": "col-sm-2", + "show": true, + "type": "label", + "default_value": "", + "options": [], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "Created" + } + ] + }, + { + "column": "DateTimeChanged", + "css_classes": "col-sm-2", + "show": true, + "type": "label", + "default_value": "", + "options": [], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "Changed" + } + ] + }, + { + "column": "Status", + "css_classes": "col-sm-1", + "show": true, + "type": "replace", + "default_value": "", + "options": [ + { + "equals": "watched-not-changed", + "replacement": "
" + }, + { + "equals": "watched-changed", + "replacement": "
" + }, + { + "equals": "new", + "replacement": "
" + }, + { + "equals": "missing-in-last-scan", + "replacement": "
" + } + ], + "localized": [ + "name" + ], + "name": [ + { + "language_code": "en_us", + "string": "Status" + } + ] + } + ] +} \ No newline at end of file diff --git a/front/plugins/ipneigh/ipneigh.py b/front/plugins/ipneigh/ipneigh.py new file mode 100755 index 000000000..4194dcee4 --- /dev/null +++ b/front/plugins/ipneigh/ipneigh.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python + +import os +import pathlib +import sys +import json +import sqlite3 +import subprocess +from datetime import datetime +from pytz import timezone +from functools import reduce + +# Define the installation path and extend the system path for plugin imports +INSTALL_PATH = "/app" +sys.path.extend([f"{INSTALL_PATH}/front/plugins", f"{INSTALL_PATH}/server"]) + +from plugin_helper import Plugin_Object, Plugin_Objects, decodeBase64, handleEmpty +from plugin_utils import get_plugins_configs +from logger import mylog +from const import pluginsPath, fullDbPath +from helper import timeNowTZ, get_setting_value +from notification import write_notification +import conf + +# Make sure the TIMEZONE for logging is correct +conf.tz = timezone(get_setting_value('TIMEZONE')) + +# Define the current path and log file paths +CUR_PATH = str(pathlib.Path(__file__).parent.resolve()) +LOG_FILE = os.path.join(CUR_PATH, 'script.log') +RESULT_FILE = os.path.join(CUR_PATH, 'last_result.log') + +# Initialize the Plugin obj output file +plugin_objects = Plugin_Objects(RESULT_FILE) + +pluginName = 'IPNEIGH' + +def main(): + mylog('verbose', [f'[{pluginName}] In script']) + + # Retrieve configuration settings + interfaces = get_setting_value('IPNEIGH_interfaces') + + mylog('verbose', [f'[{pluginName}] Interfaces value: {interfaces}']) + + # retrieve data + raw_neighbors = get_neighbors(interfaces) + + neighbors = parse_neighbors(raw_neighbors) + + #mylog('verbose', [f'[{pluginName}] Found neighbors: {neighbors}']) + + # Process the data into native application tables + if len(neighbors) > 0: + + # insert devices into the lats_result.log + # make sure the below mapping is mapped in config.json, for example: + #"database_column_definitions": [ + # { + # "column": "Object_PrimaryID", <--------- the value I save into primaryId + # "mapped_to_column": "cur_MAC", <--------- gets inserted into the CurrentScan DB table column cur_MAC + # + for device in neighbors: + plugin_objects.add_object( + primaryId = device['mac'], + secondaryId = device['ip'], + watched1 = handleEmpty(device['hostname']), # empty + watched2 = handleEmpty(device['vendor']), # empty + watched3 = handleEmpty(device['device_type']), # empty + watched4 = handleEmpty(device['last_seen']), # sometime empty + extra = '', + foreignKey = "" #device['mac'] + # helpVal1 = "Something1", # Optional Helper values to be passed for mapping into the app + # helpVal2 = "Something1", # If you need to use even only 1, add the remaining ones too + # helpVal3 = "Something1", # and set them to 'null'. Check the the docs for details: + # helpVal4 = "Something1", # https://github.com/jokob-sk/NetAlertX/blob/main/docs/PLUGINS_DEV.md + ) + + mylog('verbose', [f'[{pluginName}] New entries: "{len(neighbors)}"']) + + # log result + plugin_objects.write_result_file() + + return 0 + +def parse_neighbors(raw_neighbors: list[str]): + neighbors = [] + for line in raw_neighbors: + if "lladdr" in line: + # Known data + fields = line.split() + + if not is_multicast(fields[0]): + # mylog('verbose', [f'[{pluginName}] adding ip {fields[0]}"']) + neighbor = {} + neighbor['ip'] = fields[0] + neighbor['mac'] = fields[2] + neighbor['reachability'] = fields[3] + + # Unknown data + neighbor['hostname'] = '(unknown)' + neighbor['vendor'] = '(unknown)' + neighbor['device_type'] = '(unknown)' + + # Last seen now if reachable + if neighbor['reachability'] == "REACHABLE": + neighbor['last_seen'] = datetime.now() + else: + neighbor['last_seen'] = "" + + neighbors.append(neighbor) + + return neighbors + + +def is_multicast(ip): + prefixes = ['ff', '224', '231', '232', '233', '234', '238', '239'] + return reduce(lambda acc, prefix: acc or ip.startswith(prefix), prefixes, False) + +# retrieve data +def get_neighbors(interfaces): + + results = [] + + for interface in interfaces.split(","): + try: + + # Ping all IPv6 devices in multicast to trigger NDP + + mylog('verbose', [f'[{pluginName}] Pinging on interface: "{interface}"']) + command = f"ping ff02::1%{interface} -c 2".split() + subprocess.run(command) + mylog('verbose', [f'[{pluginName}] Pinging completed: "{interface}"']) + + # Check the neighbourhood tables + + mylog('verbose', [f'[{pluginName}] Scanning interface: "{interface}"']) + command = f"ip neighbor show nud all dev {interface}".split() + output = subprocess.check_output(command, universal_newlines=True) + results += output.split("\n") + + mylog('verbose', [f'[{pluginName}] Scanning interface succeded: "{interface}"']) + except subprocess.CalledProcessError as e: + # An error occurred, handle it + + mylog('verbose', [f'[{pluginName}] Scanning interface failed: "{interface}"']) + error_type = type(e).__name__ # Capture the error type + + return results + +if __name__ == '__main__': + main() From e6274b9f3d80aed043aa8532f66b3e0b6ea662a2 Mon Sep 17 00:00:00 2001 From: KayJay7 <31775749+KayJay7@users.noreply.github.com> Date: Tue, 26 Nov 2024 14:18:21 +0000 Subject: [PATCH 2/4] Added IPNEIGH to the plugins README.md file --- front/plugins/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/front/plugins/README.md b/front/plugins/README.md index ee13e0643..f3852f37d 100755 --- a/front/plugins/README.md +++ b/front/plugins/README.md @@ -58,6 +58,7 @@ Device-detecting plugins insert values into the `CurrentScan` database table. T | `VNDRPDT` | ⚙ | Vendor database update | | | Script | [vendor_update](/front/plugins/vendor_update/) | | `WEBHOOK` | ▶️ | Webhook notifications | | | Script | [_publisher_webhook](/front/plugins/_publisher_webhook/) | | `WEBMON` | ♻ | Website down monitoring | | | Script | [website_monitor](/front/plugins/website_monitor/) | +| `IPNEIGH` | 🔍 | Scan ARP (IPv4) and NDP (IPv6) tables | | | Script | [ipneigh](/front/plugins/ipneigh/) | > \* The database cleanup plugin (`DBCLNP`) is not _required_ but the app will become unusable after a while if not executed. From d92ebc24de4495a2c192e213598505dfc6779ec9 Mon Sep 17 00:00:00 2001 From: KayJay7 <31775749+KayJay7@users.noreply.github.com> Date: Tue, 26 Nov 2024 14:37:17 +0000 Subject: [PATCH 3/4] Fixed note in IPNEIGH README.md --- front/plugins/ipneigh/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/plugins/ipneigh/README.md b/front/plugins/ipneigh/README.md index dca94ddf7..c65ea0500 100755 --- a/front/plugins/ipneigh/README.md +++ b/front/plugins/ipneigh/README.md @@ -19,4 +19,4 @@ To set up the plugin correctly, make sure to add in the plugin settings the name ### Notes - `ARPSCAN` does a better job at discovering IPv4 devices because it explicitly sends arp requests -- IPv6 devices can \ No newline at end of file +- IPv6 devices will often have multiple addresses, but the ping answer will contain only one. This means that in general this plugin will not discover every address but only those who answer \ No newline at end of file From e34281045dd94b324e55b97a1c734d331de12c01 Mon Sep 17 00:00:00 2001 From: KayJay7 <31775749+KayJay7@users.noreply.github.com> Date: Thu, 28 Nov 2024 10:14:22 +0000 Subject: [PATCH 4/4] Fixed offline detection in IPNEIGH --- front/plugins/ipneigh/README.md | 2 +- front/plugins/ipneigh/config.json | 2 +- front/plugins/ipneigh/ipneigh.py | 29 ++++++++--------------------- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/front/plugins/ipneigh/README.md b/front/plugins/ipneigh/README.md index c65ea0500..715ccce94 100755 --- a/front/plugins/ipneigh/README.md +++ b/front/plugins/ipneigh/README.md @@ -14,7 +14,7 @@ To set up the plugin correctly, make sure to add in the plugin settings the name ### Usage - Head to **Settings** > **IP Neigh** to add the interfaces you want to scan to the `IPNEIGH_interfaces` option -- The interface list must be formatted without whitespaces and comma separated +- The interface list must be formatted without whitespaces and comma separated e.g. `eth0,wl1,tap0` ### Notes diff --git a/front/plugins/ipneigh/config.json b/front/plugins/ipneigh/config.json index 64b73ead5..93c2059e3 100755 --- a/front/plugins/ipneigh/config.json +++ b/front/plugins/ipneigh/config.json @@ -126,7 +126,7 @@ ] }, "maxLength": 150, - "default_value": "", + "default_value": "eth0", "options": [], "localized": [ "name", diff --git a/front/plugins/ipneigh/ipneigh.py b/front/plugins/ipneigh/ipneigh.py index 4194dcee4..467da993f 100755 --- a/front/plugins/ipneigh/ipneigh.py +++ b/front/plugins/ipneigh/ipneigh.py @@ -47,27 +47,20 @@ def main(): raw_neighbors = get_neighbors(interfaces) neighbors = parse_neighbors(raw_neighbors) - - #mylog('verbose', [f'[{pluginName}] Found neighbors: {neighbors}']) # Process the data into native application tables if len(neighbors) > 0: - # insert devices into the lats_result.log - # make sure the below mapping is mapped in config.json, for example: - #"database_column_definitions": [ - # { - # "column": "Object_PrimaryID", <--------- the value I save into primaryId - # "mapped_to_column": "cur_MAC", <--------- gets inserted into the CurrentScan DB table column cur_MAC - # for device in neighbors: plugin_objects.add_object( primaryId = device['mac'], secondaryId = device['ip'], - watched1 = handleEmpty(device['hostname']), # empty - watched2 = handleEmpty(device['vendor']), # empty - watched3 = handleEmpty(device['device_type']), # empty - watched4 = handleEmpty(device['last_seen']), # sometime empty + watched4 = device['last_seen'], + + # The following are always unknown + watched1 = device['hostname'], # don't use these --> handleEmpty(device['hostname']), + watched2 = device['vendor'], # handleEmpty(device['vendor']), + watched3 = device['device_type'], # handleEmpty(device['device_type']), extra = '', foreignKey = "" #device['mac'] # helpVal1 = "Something1", # Optional Helper values to be passed for mapping into the app @@ -86,7 +79,7 @@ def main(): def parse_neighbors(raw_neighbors: list[str]): neighbors = [] for line in raw_neighbors: - if "lladdr" in line: + if "lladdr" in line and "REACHABLE" in line: # Known data fields = line.split() @@ -95,18 +88,12 @@ def parse_neighbors(raw_neighbors: list[str]): neighbor = {} neighbor['ip'] = fields[0] neighbor['mac'] = fields[2] - neighbor['reachability'] = fields[3] + neighbor['last_seen'] = datetime.now() # Unknown data neighbor['hostname'] = '(unknown)' neighbor['vendor'] = '(unknown)' neighbor['device_type'] = '(unknown)' - - # Last seen now if reachable - if neighbor['reachability'] == "REACHABLE": - neighbor['last_seen'] = datetime.now() - else: - neighbor['last_seen'] = "" neighbors.append(neighbor)