Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(dpi): adding more filtering options #949

Merged
merged 1 commit into from
Dec 5, 2024
Merged

feat(dpi): adding more filtering options #949

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 102 additions & 0 deletions packages/ns-api/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -799,6 +799,108 @@ api-cli ns.dpireport summary-by-client --data '{"year": "2023", "month": "06", "

It has the same structure of the `summary` API.

### summary-v2

Improved version of the `summary` API, the api can accept the following parameters:

- year: the year of the report (default to the current year)
- month: the month of the report (default to the current month)
- day: the day of the report (default to the current day)
- client: filter data for the given client
- section: section to filter data (application, protocol, host)
- value: value to filter the previous section
- limit: limit the items returned in the lists (default to 20)

Example:

```
api-cli ns.dpireport summary-v2 --data '{"year": "2023", "month": "06", "day": "16", "limit": 10}'
```

Response:

```json
{
"total_traffic": 16035446620,
"hourly_traffic": [
{
"id": "00",
"traffic": 120281002
},
{
"id": "01",
"traffic": 879624129
},
{
"id": "02",
"traffic": 127278297
},
{
"id": "03",
"traffic": 320193579
}
],
"clients": [
{
"id": "11.0.1.1",
"label": "11.0.1.1",
"traffic": 1962950942
},
{
"id": "10.0.0.2",
"label": "cool-PC",
"traffic": 1413666916
}
],
"applications": [
{
"id": "unknown",
"label": "Unknown",
"traffic": 2148481014
},
{
"id": "netify.google",
"label": "Google",
"traffic": 2012854079
},
{
"id": "netify.youtube",
"label": "Youtube",
"traffic": 732979058
}
],
"remote_hosts": [
{
"id": "nethservice.nethesis.it",
"traffic": 1142530419
},
{
"id": "community.nethserver.org",
"traffic": 389680308
},
{
"id": "1d.tlu.dl.delivery.mp.microsoft.com",
"traffic": 296533256
}
],
"protocols": [
{
"id": "http/s",
"label": "HTTP/S",
"traffic": 9277455653
},
{
"id": "quic",
"label": "QUIC",
"traffic": 3638009875
}
]
}
```

Note: the fields `applications`, `remote_hosts` and `protocols` will always be present if no section is specified. If
a section is specified, the response will contain only the `clients`, `hourly_traffic` and `total_traffic` fields.

## ns.ovpntunnel

### list-tunnels
Expand Down
118 changes: 69 additions & 49 deletions packages/ns-api/files/ns.dpireport
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ def _extract_data(dpi_file: str):
return dict()


def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_application=None, limit=20):
def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_section=None, narrow_value=None, limit=20):
if year is None:
year = f'{date.today().year:02}'
if month is None:
Expand All @@ -135,6 +135,8 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli

total_traffic = 0
raw_hourly_traffic = dict[str, int]()
for i in range(24):
raw_hourly_traffic[f'{i:02}'] = 0
raw_applications = dict[str, int]()
raw_clients = list[dict]()
raw_remote_hosts = dict[str, int]()
Expand All @@ -148,32 +150,53 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli

for time in data[client]:
for application in data[client][time]['application']:
# application
if narrow_application is not None and application != narrow_application:
if narrow_section == 'application' and application != narrow_value:
continue
elif narrow_section is not None and narrow_section != 'application':
break
if application not in raw_applications:
raw_applications[application] = 0
raw_applications[application] += data[client][time]['application'][application]
# total traffic
total_traffic += data[client][time]['application'][application]
# hourly traffic
if time not in raw_hourly_traffic:
raw_hourly_traffic[time] = 0
raw_hourly_traffic[time] += data[client][time]['application'][application]
# client total traffic
raw_client_total_traffic += data[client][time]['application'][application]

if narrow_application is None:
# remote hosts
for host in data[client][time]['host']:
if host not in raw_remote_hosts:
raw_remote_hosts[host] = 0
raw_remote_hosts[host] += data[client][time]['host'][host]
# protocols
for protocol in data[client][time]['protocol']:
if protocol not in raw_protocols:
raw_protocols[protocol] = 0
raw_protocols[protocol] += data[client][time]['protocol'][protocol]
for host in data[client][time]['host']:
if narrow_section == 'host' and host != narrow_value:
continue
elif narrow_section is not None and narrow_section != 'host':
break
if host not in raw_remote_hosts:
raw_remote_hosts[host] = 0
raw_remote_hosts[host] += data[client][time]['host'][host]
for protocol in data[client][time]['protocol']:
if narrow_section == 'protocol' and protocol != narrow_value:
continue
elif narrow_section is not None and narrow_section != 'protocol':
break
if protocol not in raw_protocols:
raw_protocols[protocol] = 0
raw_protocols[protocol] += data[client][time]['protocol'][protocol]

match narrow_section:
case 'host':
if narrow_value not in data[client][time]['host']:
continue
total_traffic += data[client][time]['host'][narrow_value]
raw_hourly_traffic[time] += data[client][time]['host'][narrow_value]
raw_client_total_traffic += data[client][time]['host'][narrow_value]
case 'protocol':
if narrow_value not in data[client][time]['protocol']:
continue
total_traffic += data[client][time]['protocol'][narrow_value]
raw_hourly_traffic[time] += data[client][time]['protocol'][narrow_value]
raw_client_total_traffic += data[client][time]['protocol'][narrow_value]
case 'application':
if narrow_value not in data[client][time]['application']:
continue
total_traffic += data[client][time]['application'][narrow_value]
raw_hourly_traffic[time] += data[client][time]['application'][narrow_value]
raw_client_total_traffic += data[client][time]['application'][narrow_value]
case _:
total_traffic += data[client][time]['total']
raw_hourly_traffic[time] += data[client][time]['total']
raw_client_total_traffic += data[client][time]['total']

# append client
raw_clients.append({
Expand All @@ -185,21 +208,6 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli
raw_clients.sort(key=lambda x: x['traffic'], reverse=True)
final_clients = raw_clients[:limit]

final_applications = list()
for item in raw_applications:
label = item
if item == 'unknown':
label = 'Unknown'
else:
label = label.removeprefix('netify.').capitalize()
final_applications.append({
'id': item,
'label': label,
'traffic': raw_applications[item]
})
final_applications.sort(key=lambda x: x['traffic'], reverse=True)
final_applications = final_applications[:limit]

final_hourly_traffic = list()
for item in raw_hourly_traffic:
final_hourly_traffic.append({
Expand All @@ -211,23 +219,36 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli
response = {
'total_traffic': total_traffic,
'hourly_traffic': final_hourly_traffic,
'applications': final_applications,
'clients': final_clients,
}

if narrow_application is None:
# remote hosts
if len(raw_applications) > 0:
final_applications = list()
for item in raw_applications:
label = item
if item == 'unknown':
label = 'Unknown'
else:
label = label.removeprefix('netify.').capitalize()
final_applications.append({
'id': item,
'label': label,
'traffic': raw_applications[item]
})
final_applications.sort(key=lambda x: x['traffic'], reverse=True)
response['applications'] = final_applications[:limit]

if len(raw_remote_hosts) > 0:
final_remote_hosts = list()
for item in raw_remote_hosts:
final_remote_hosts.append({
'id': item,
'traffic': raw_remote_hosts[item]
})
final_remote_hosts.sort(key=lambda x: x['traffic'], reverse=True)
final_remote_hosts = final_remote_hosts[:limit]
response['remote_hosts'] = final_remote_hosts
response['remote_hosts'] = final_remote_hosts[:limit]

# protocols
if len(raw_protocols) > 0:
final_protocols = list()
for item in raw_protocols:
final_protocols.append({
Expand All @@ -236,8 +257,7 @@ def summary_v2(year=None, month=None, day=None, narrow_client=None, narrow_appli
'traffic': raw_protocols[item]
})
final_protocols.sort(key=lambda x: x['traffic'], reverse=True)
final_protocols = final_protocols[:limit]
response['protocols'] = final_protocols
response['protocols'] = final_protocols[:limit]

return response

Expand All @@ -250,8 +270,8 @@ if cmd == 'list':
"summary-by-client": {"year": "2023", "month": "06", "day": "02", "client": "192.168.1.1", "limit": 10},
"details": {"year": "2023", "month": "06", "day": "16", "client": "192.168.100.22"},
"days": {},
"summary-v2": {"year": "2024", "month": "06", "day": "02", "client": "127.0.0.1", "application": "netify.http",
"limit": 20}
"summary-v2": {"year": "2024", "month": "06", "day": "02", "client": "127.0.0.1", "section": "application",
"value": "netify.http", "limit": 20}
}))
else:
action = sys.argv[2]
Expand All @@ -260,7 +280,7 @@ else:
elif action == 'summary-v2':
args = json.loads(sys.stdin.read())
print(json.dumps(summary_v2(args.get('year'), args.get('month'), args.get('day'), args.get('client'),
args.get('application'), args.get('limit', 20))))
args.get('section'), args.get('value'), args.get('limit', 20))))
else:
args = json.loads(sys.stdin.read())
year = args.get('year', f'{date.today().year:02}')
Expand Down
Loading