From ee82633eee7af102bff963cca8df7583392275f9 Mon Sep 17 00:00:00 2001 From: Tommaso Bailetti Date: Tue, 3 Dec 2024 16:37:01 +0100 Subject: [PATCH] feat(dpi): added new filtering of summary --- packages/ns-api/README.md | 102 +++++++++++++++++++++++++ packages/ns-api/files/ns.dpireport | 118 +++++++++++++++++------------ 2 files changed, 171 insertions(+), 49 deletions(-) diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index 6e65c6b7..606c2ca5 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -799,6 +799,108 @@ api-cli ns.dpireport summary-by-client --data '{"year": "2023", "month": "06", " It has the same structure of the `summary` API. +### summary-v2 + +Improved version of the `summary` API, the api can accept the following parameters: + +- year: the year of the report (default to the current year) +- month: the month of the report (default to the current month) +- day: the day of the report (default to the current day) +- client: filter data for the given client +- section: section to filter data (application, protocol, host) +- value: value to filter the previous section +- limit: limit the items returned in the lists (default to 20) + +Example: + +``` +api-cli ns.dpireport summary-v2 --data '{"year": "2023", "month": "06", "day": "16", "limit": 10}' +``` + +Response: + +```json +{ + "total_traffic": 16035446620, + "hourly_traffic": [ + { + "id": "00", + "traffic": 120281002 + }, + { + "id": "01", + "traffic": 879624129 + }, + { + "id": "02", + "traffic": 127278297 + }, + { + "id": "03", + "traffic": 320193579 + } + ], + "clients": [ + { + "id": "11.0.1.1", + "label": "11.0.1.1", + "traffic": 1962950942 + }, + { + "id": "10.0.0.2", + "label": "cool-PC", + "traffic": 1413666916 + } + ], + "applications": [ + { + "id": "unknown", + "label": "Unknown", + "traffic": 2148481014 + }, + { + "id": "netify.google", + "label": "Google", + "traffic": 2012854079 + }, + { + "id": "netify.youtube", + "label": "Youtube", + "traffic": 732979058 + } + ], + "remote_hosts": [ + { + "id": "nethservice.nethesis.it", + "traffic": 1142530419 + }, + { + "id": "community.nethserver.org", + "traffic": 389680308 + }, + { + "id": "1d.tlu.dl.delivery.mp.microsoft.com", + "traffic": 296533256 + } + ], + "protocols": [ + { + "id": "http/s", + "label": "HTTP/S", + "traffic": 9277455653 + }, + { + "id": "quic", + "label": "QUIC", + "traffic": 3638009875 + } + ] +} +``` + +Note: the fields `applications`, `remote_hosts` and `protocols` will always be present if no section is specified. If +a section is specified, the response will contain only the `clients`, `hourly_traffic` and `total_traffic` fields. + ## ns.ovpntunnel ### list-tunnels diff --git a/packages/ns-api/files/ns.dpireport b/packages/ns-api/files/ns.dpireport index 5289d51d..d76e978b 100755 --- a/packages/ns-api/files/ns.dpireport +++ b/packages/ns-api/files/ns.dpireport @@ -124,7 +124,7 @@ def _extract_data(dpi_file: str): return dict() -def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_application=None, limit=20): +def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_section=None, narrow_value=None, limit=20): if year is None: year = f'{date.today().year:02}' if month is None: @@ -135,6 +135,8 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli total_traffic = 0 raw_hourly_traffic = dict[str, int]() + for i in range(24): + raw_hourly_traffic[f'{i:02}'] = 0 raw_applications = dict[str, int]() raw_clients = list[dict]() raw_remote_hosts = dict[str, int]() @@ -148,32 +150,53 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli for time in data[client]: for application in data[client][time]['application']: - # application - if narrow_application is not None and application != narrow_application: + if narrow_section == 'application' and application != narrow_value: continue + elif narrow_section is not None and narrow_section != 'application': + break if application not in raw_applications: raw_applications[application] = 0 raw_applications[application] += data[client][time]['application'][application] - # total traffic - total_traffic += data[client][time]['application'][application] - # hourly traffic - if time not in raw_hourly_traffic: - raw_hourly_traffic[time] = 0 - raw_hourly_traffic[time] += data[client][time]['application'][application] - # client total traffic - raw_client_total_traffic += data[client][time]['application'][application] - - if narrow_application is None: - # remote hosts - for host in data[client][time]['host']: - if host not in raw_remote_hosts: - raw_remote_hosts[host] = 0 - raw_remote_hosts[host] += data[client][time]['host'][host] - # protocols - for protocol in data[client][time]['protocol']: - if protocol not in raw_protocols: - raw_protocols[protocol] = 0 - raw_protocols[protocol] += data[client][time]['protocol'][protocol] + for host in data[client][time]['host']: + if narrow_section == 'host' and host != narrow_value: + continue + elif narrow_section is not None and narrow_section != 'host': + break + if host not in raw_remote_hosts: + raw_remote_hosts[host] = 0 + raw_remote_hosts[host] += data[client][time]['host'][host] + for protocol in data[client][time]['protocol']: + if narrow_section == 'protocol' and protocol != narrow_value: + continue + elif narrow_section is not None and narrow_section != 'protocol': + break + if protocol not in raw_protocols: + raw_protocols[protocol] = 0 + raw_protocols[protocol] += data[client][time]['protocol'][protocol] + + match narrow_section: + case 'host': + if narrow_value not in data[client][time]['host']: + continue + total_traffic += data[client][time]['host'][narrow_value] + raw_hourly_traffic[time] += data[client][time]['host'][narrow_value] + raw_client_total_traffic += data[client][time]['host'][narrow_value] + case 'protocol': + if narrow_value not in data[client][time]['protocol']: + continue + total_traffic += data[client][time]['protocol'][narrow_value] + raw_hourly_traffic[time] += data[client][time]['protocol'][narrow_value] + raw_client_total_traffic += data[client][time]['protocol'][narrow_value] + case 'application': + if narrow_value not in data[client][time]['application']: + continue + total_traffic += data[client][time]['application'][narrow_value] + raw_hourly_traffic[time] += data[client][time]['application'][narrow_value] + raw_client_total_traffic += data[client][time]['application'][narrow_value] + case _: + total_traffic += data[client][time]['total'] + raw_hourly_traffic[time] += data[client][time]['total'] + raw_client_total_traffic += data[client][time]['total'] # append client raw_clients.append({ @@ -185,21 +208,6 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli raw_clients.sort(key=lambda x: x['traffic'], reverse=True) final_clients = raw_clients[:limit] - final_applications = list() - for item in raw_applications: - label = item - if item == 'unknown': - label = 'Unknown' - else: - label = label.removeprefix('netify.').capitalize() - final_applications.append({ - 'id': item, - 'label': label, - 'traffic': raw_applications[item] - }) - final_applications.sort(key=lambda x: x['traffic'], reverse=True) - final_applications = final_applications[:limit] - final_hourly_traffic = list() for item in raw_hourly_traffic: final_hourly_traffic.append({ @@ -211,12 +219,26 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli response = { 'total_traffic': total_traffic, 'hourly_traffic': final_hourly_traffic, - 'applications': final_applications, 'clients': final_clients, } - if narrow_application is None: - # remote hosts + if len(raw_applications) > 0: + final_applications = list() + for item in raw_applications: + label = item + if item == 'unknown': + label = 'Unknown' + else: + label = label.removeprefix('netify.').capitalize() + final_applications.append({ + 'id': item, + 'label': label, + 'traffic': raw_applications[item] + }) + final_applications.sort(key=lambda x: x['traffic'], reverse=True) + response['applications'] = final_applications[:limit] + + if len(raw_remote_hosts) > 0: final_remote_hosts = list() for item in raw_remote_hosts: final_remote_hosts.append({ @@ -224,10 +246,9 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli 'traffic': raw_remote_hosts[item] }) final_remote_hosts.sort(key=lambda x: x['traffic'], reverse=True) - final_remote_hosts = final_remote_hosts[:limit] - response['remote_hosts'] = final_remote_hosts + response['remote_hosts'] = final_remote_hosts[:limit] - # protocols + if len(raw_protocols) > 0: final_protocols = list() for item in raw_protocols: final_protocols.append({ @@ -236,8 +257,7 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli 'traffic': raw_protocols[item] }) final_protocols.sort(key=lambda x: x['traffic'], reverse=True) - final_protocols = final_protocols[:limit] - response['protocols'] = final_protocols + response['protocols'] = final_protocols[:limit] return response @@ -250,8 +270,8 @@ if cmd == 'list': "summary-by-client": {"year": "2023", "month": "06", "day": "02", "client": "192.168.1.1", "limit": 10}, "details": {"year": "2023", "month": "06", "day": "16", "client": "192.168.100.22"}, "days": {}, - "summary-v2": {"year": "2024", "month": "06", "day": "02", "client": "127.0.0.1", "application": "netify.http", - "limit": 20} + "summary-v2": {"year": "2024", "month": "06", "day": "02", "client": "127.0.0.1", "section": "application", + "value": "netify.http", "limit": 20} })) else: action = sys.argv[2] @@ -260,7 +280,7 @@ else: elif action == 'summary-v2': args = json.loads(sys.stdin.read()) print(json.dumps(summary_v2(args.get('year'), args.get('month'), args.get('day'), args.get('client'), - args.get('application'), args.get('limit', 20)))) + args.get('section'), args.get('value'), args.get('limit', 20)))) else: args = json.loads(sys.stdin.read()) year = args.get('year', f'{date.today().year:02}')