Skip to content

Commit

Permalink
feat(dpi): added new filtering of summary (#949)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Tbaile
Tbaile authored Dec 5, 2024
1 parent a2de0d8 commit 7f72651
Show file tree
Hide file tree
Showing 2 changed files with 171 additions and 49 deletions.
102 changes: 102 additions & 0 deletions packages/ns-api/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -799,6 +799,108 @@ api-cli ns.dpireport summary-by-client --data '{"year": "2023", "month": "06", "

It has the same structure of the `summary` API.

### summary-v2

Improved version of the `summary` API, the api can accept the following parameters:

- year: the year of the report (default to the current year)
- month: the month of the report (default to the current month)
- day: the day of the report (default to the current day)
- client: filter data for the given client
- section: section to filter data (application, protocol, host)
- value: value to filter the previous section
- limit: limit the items returned in the lists (default to 20)

Example:

```
api-cli ns.dpireport summary-v2 --data '{"year": "2023", "month": "06", "day": "16", "limit": 10}'
```

Response:

```json
{
"total_traffic": 16035446620,
"hourly_traffic": [
{
"id": "00",
"traffic": 120281002
},
{
"id": "01",
"traffic": 879624129
},
{
"id": "02",
"traffic": 127278297
},
{
"id": "03",
"traffic": 320193579
}
],
"clients": [
{
"id": "11.0.1.1",
"label": "11.0.1.1",
"traffic": 1962950942
},
{
"id": "10.0.0.2",
"label": "cool-PC",
"traffic": 1413666916
}
],
"applications": [
{
"id": "unknown",
"label": "Unknown",
"traffic": 2148481014
},
{
"id": "netify.google",
"label": "Google",
"traffic": 2012854079
},
{
"id": "netify.youtube",
"label": "Youtube",
"traffic": 732979058
}
],
"remote_hosts": [
{
"id": "nethservice.nethesis.it",
"traffic": 1142530419
},
{
"id": "community.nethserver.org",
"traffic": 389680308
},
{
"id": "1d.tlu.dl.delivery.mp.microsoft.com",
"traffic": 296533256
}
],
"protocols": [
{
"id": "http/s",
"label": "HTTP/S",
"traffic": 9277455653
},
{
"id": "quic",
"label": "QUIC",
"traffic": 3638009875
}
]
}
```

Note: the fields `applications`, `remote_hosts` and `protocols` will always be present if no section is specified. If
a section is specified, the response will contain only the `clients`, `hourly_traffic` and `total_traffic` fields.

## ns.ovpntunnel

### list-tunnels
Expand Down
118 changes: 69 additions & 49 deletions packages/ns-api/files/ns.dpireport
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ def _extract_data(dpi_file: str):
return dict()


def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_application=None, limit=20):
def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_section=None, narrow_value=None, limit=20):
if year is None:
year = f'{date.today().year:02}'
if month is None:
Expand All @@ -135,6 +135,8 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli

total_traffic = 0
raw_hourly_traffic = dict[str, int]()
for i in range(24):
raw_hourly_traffic[f'{i:02}'] = 0
raw_applications = dict[str, int]()
raw_clients = list[dict]()
raw_remote_hosts = dict[str, int]()
Expand All @@ -148,32 +150,53 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli

for time in data[client]:
for application in data[client][time]['application']:
# application
if narrow_application is not None and application != narrow_application:
if narrow_section == 'application' and application != narrow_value:
continue
elif narrow_section is not None and narrow_section != 'application':
break
if application not in raw_applications:
raw_applications[application] = 0
raw_applications[application] += data[client][time]['application'][application]
# total traffic
total_traffic += data[client][time]['application'][application]
# hourly traffic
if time not in raw_hourly_traffic:
raw_hourly_traffic[time] = 0
raw_hourly_traffic[time] += data[client][time]['application'][application]
# client total traffic
raw_client_total_traffic += data[client][time]['application'][application]

if narrow_application is None:
# remote hosts
for host in data[client][time]['host']:
if host not in raw_remote_hosts:
raw_remote_hosts[host] = 0
raw_remote_hosts[host] += data[client][time]['host'][host]
# protocols
for protocol in data[client][time]['protocol']:
if protocol not in raw_protocols:
raw_protocols[protocol] = 0
raw_protocols[protocol] += data[client][time]['protocol'][protocol]
for host in data[client][time]['host']:
if narrow_section == 'host' and host != narrow_value:
continue
elif narrow_section is not None and narrow_section != 'host':
break
if host not in raw_remote_hosts:
raw_remote_hosts[host] = 0
raw_remote_hosts[host] += data[client][time]['host'][host]
for protocol in data[client][time]['protocol']:
if narrow_section == 'protocol' and protocol != narrow_value:
continue
elif narrow_section is not None and narrow_section != 'protocol':
break
if protocol not in raw_protocols:
raw_protocols[protocol] = 0
raw_protocols[protocol] += data[client][time]['protocol'][protocol]

match narrow_section:
case 'host':
if narrow_value not in data[client][time]['host']:
continue
total_traffic += data[client][time]['host'][narrow_value]
raw_hourly_traffic[time] += data[client][time]['host'][narrow_value]
raw_client_total_traffic += data[client][time]['host'][narrow_value]
case 'protocol':
if narrow_value not in data[client][time]['protocol']:
continue
total_traffic += data[client][time]['protocol'][narrow_value]
raw_hourly_traffic[time] += data[client][time]['protocol'][narrow_value]
raw_client_total_traffic += data[client][time]['protocol'][narrow_value]
case 'application':
if narrow_value not in data[client][time]['application']:
continue
total_traffic += data[client][time]['application'][narrow_value]
raw_hourly_traffic[time] += data[client][time]['application'][narrow_value]
raw_client_total_traffic += data[client][time]['application'][narrow_value]
case _:
total_traffic += data[client][time]['total']
raw_hourly_traffic[time] += data[client][time]['total']
raw_client_total_traffic += data[client][time]['total']

# append client
raw_clients.append({
Expand All @@ -185,21 +208,6 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli
raw_clients.sort(key=lambda x: x['traffic'], reverse=True)
final_clients = raw_clients[:limit]

final_applications = list()
for item in raw_applications:
label = item
if item == 'unknown':
label = 'Unknown'
else:
label = label.removeprefix('netify.').capitalize()
final_applications.append({
'id': item,
'label': label,
'traffic': raw_applications[item]
})
final_applications.sort(key=lambda x: x['traffic'], reverse=True)
final_applications = final_applications[:limit]

final_hourly_traffic = list()
for item in raw_hourly_traffic:
final_hourly_traffic.append({
Expand All @@ -211,23 +219,36 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli
response = {
'total_traffic': total_traffic,
'hourly_traffic': final_hourly_traffic,
'applications': final_applications,
'clients': final_clients,
}

if narrow_application is None:
# remote hosts
if len(raw_applications) > 0:
final_applications = list()
for item in raw_applications:
label = item
if item == 'unknown':
label = 'Unknown'
else:
label = label.removeprefix('netify.').capitalize()
final_applications.append({
'id': item,
'label': label,
'traffic': raw_applications[item]
})
final_applications.sort(key=lambda x: x['traffic'], reverse=True)
response['applications'] = final_applications[:limit]

if len(raw_remote_hosts) > 0:
final_remote_hosts = list()
for item in raw_remote_hosts:
final_remote_hosts.append({
'id': item,
'traffic': raw_remote_hosts[item]
})
final_remote_hosts.sort(key=lambda x: x['traffic'], reverse=True)
final_remote_hosts = final_remote_hosts[:limit]
response['remote_hosts'] = final_remote_hosts
response['remote_hosts'] = final_remote_hosts[:limit]

# protocols
if len(raw_protocols) > 0:
final_protocols = list()
for item in raw_protocols:
final_protocols.append({
Expand All @@ -236,8 +257,7 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli
'traffic': raw_protocols[item]
})
final_protocols.sort(key=lambda x: x['traffic'], reverse=True)
final_protocols = final_protocols[:limit]
response['protocols'] = final_protocols
response['protocols'] = final_protocols[:limit]

return response

Expand All @@ -250,8 +270,8 @@ if cmd == 'list':
"summary-by-client": {"year": "2023", "month": "06", "day": "02", "client": "192.168.1.1", "limit": 10},
"details": {"year": "2023", "month": "06", "day": "16", "client": "192.168.100.22"},
"days": {},
"summary-v2": {"year": "2024", "month": "06", "day": "02", "client": "127.0.0.1", "application": "netify.http",
"limit": 20}
"summary-v2": {"year": "2024", "month": "06", "day": "02", "client": "127.0.0.1", "section": "application",
"value": "netify.http", "limit": 20}
}))
else:
action = sys.argv[2]
Expand All @@ -260,7 +280,7 @@ else:
elif action == 'summary-v2':
args = json.loads(sys.stdin.read())
print(json.dumps(summary_v2(args.get('year'), args.get('month'), args.get('day'), args.get('client'),
args.get('application'), args.get('limit', 20))))
args.get('section'), args.get('value'), args.get('limit', 20))))
else:
args = json.loads(sys.stdin.read())
year = args.get('year', f'{date.today().year:02}')
Expand Down

0 comments on commit 7f72651

Please sign in to comment.