Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improved suspicious http user agent detection. #1537

Merged
merged 1 commit into from
May 2, 2022
Merged

Improved suspicious http user agent detection. #1537

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 36 additions & 11 deletions src/lib/protocols/http.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,19 +420,44 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp

static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
char *ua) {
u_int len;
char const *ua, size_t ua_len) {
char *double_slash;

if((!ua) || (ua[0] == '\0'))
return;
else
len = strlen(ua);

if (ua_len > 12)
{
size_t i, upper_case_count = 0;

for (i = 0; i < ua_len; ++i)
{
/*
* We assume at least one non alpha char.
* e.g. ' ', '-' or ';' ...
*/
if (isalpha(ua[i]) == 0)
{
break;
}
if (isupper(ua[i]) != 0)
{
upper_case_count++;
}
}

if (i == ua_len)
{
float upper_case_ratio = (float)upper_case_count / (float)ua_len;
if (upper_case_ratio >= 0.2f)
{
ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);
}
}
}

if((!strncmp(ua, "<?", 2))
|| strchr(ua, '$')
// || ndpi_check_dga_name(ndpi_struct, NULL, ua, 0)
// || ndpi_match_bigram(ndpi_struct, &ndpi_struct->impossible_bigrams_automa, ua)
)
ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);

Expand All @@ -448,8 +473,8 @@ static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_stru
if(!strncmp(ua, "jndi:ldap://", 12)) /* Log4J */ {
ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT);
} else if(
(len < 4) /* Too short */
|| (len > 256) /* Too long */
(ua_len < 4) /* Too short */
|| (ua_len > 256) /* Too long */
|| (!strncmp(ua, "test", 4))
|| strchr(ua, '{')
|| strchr(ua, '}')
Expand All @@ -464,8 +489,8 @@ static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_stru
*/
if((strstr(ua, "+http") != NULL)
|| (strstr(ua, " http") != NULL)
|| ndpi_strncasestr(ua, "Crawler", strlen(ua))
|| ndpi_strncasestr(ua, "Bot", strlen(ua)) /* bot/robot */
|| ndpi_strncasestr(ua, "Crawler", ua_len)
|| ndpi_strncasestr(ua, "Bot", ua_len) /* bot/robot */
) {
ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_CRAWLER_BOT);
}
Expand Down Expand Up @@ -537,7 +562,7 @@ int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,

if (ndpi_user_agent_set(flow, ua_ptr, ua_ptr_len) != NULL)
{
ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent);
ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent, ua_ptr_len);
} else {
NDPI_LOG_DBG2(ndpi_struct, "Could not set HTTP user agent\n");
}
Expand Down
Binary file added tests/pcap/emotet.pcap
View file
Open in desktop
Binary file not shown.
20 changes: 20 additions & 0 deletions tests/result/emotet.pcap.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
Guessed flow protos: 1

DPI Packets (TCP): 121 (20.17 pkts/flow)
Confidence DPI : 6 (flows)

SMTP 626 438465 1
HTTP 1601 1581542 3
TLS 153 107018 2

JA3 Host Stats:
IP Address # JA3C
1 10.4.25.101 1


1 TCP 10.4.20.102:54319 <-> 107.161.178.210:80 [proto: 7/HTTP][ClearText][Confidence: DPI][cat: Web/5][272 pkts/16545 bytes <-> 557 pkts/800118 bytes][Goodput ratio: 1/96][9.12 sec][Hostname/SNI: gandhitoday.org][bytes ratio: -0.959 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 26/11 2171/1215 155/59][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 61/1436 279/1442 13/84][URL: gandhitoday.org/video/6JvA8/][StatusCode: 200][User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; Trident/7.0; rv:11.0) like Gecko][Risk: ** Binary App Transfer **][Risk Score: 250][PLAIN TEXT (GET /video/6J)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0]
2 TCP 10.4.25.101:49797 <-> 77.105.36.156:80 [proto: 7/HTTP][ClearText][Confidence: DPI][cat: Download/7][169 pkts/10292 bytes <-> 395 pkts/565664 bytes][Goodput ratio: 1/96][1.99 sec][Hostname/SNI: filmmogzivota.rs][bytes ratio: -0.964 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/4 292/171 38/19][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 61/1432 206/1442 11/107][URL: filmmogzivota.rs/SpryAssets/gDR/][StatusCode: 200][Content-Type: application/x-msdownload][User-Agent: vBKbaQgjyvRRbcgfvlsc][Risk: ** Binary App Transfer **** HTTP Susp User-Agent **][Risk Score: 350][PLAIN TEXT (GET /SpryAssets/gDR/ HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0]
3 TCP 10.2.25.102:57309 <-> 193.252.22.84:587 [proto: 3/SMTP][ClearText][Confidence: DPI][cat: Email/3][303 pkts/420177 bytes <-> 323 pkts/18288 bytes][Goodput ratio: 96/5][19.04 sec][Hostname/SNI: opmta1mto02nd1][bytes ratio: 0.917 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 26/66 1205/3211 138/351][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 1387/57 1514/214 400/13][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][PLAIN TEXT (220 opmta)][Plen Bins: 7,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,83,0,0]
4 TCP 10.3.29.101:56309 <-> 104.161.127.22:80 [proto: 7/HTTP][ClearText][Confidence: DPI][cat: Web/5][72 pkts/4883 bytes <-> 136 pkts/184040 bytes][Goodput ratio: 20/96][11.81 sec][Hostname/SNI: fkl.co.ke][bytes ratio: -0.948 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 224/98 7597/7597 1122/760][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 68/1353 591/1415 81/273][URL: fkl.co.ke/wp-content/Elw3kPvOsZxM5/][StatusCode: 200][Content-Type: text/html][User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.55][PLAIN TEXT (GET /wp)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0]
5 TCP 10.4.25.101:49803 <-> 138.197.147.101:443 [proto: 91/TLS][Encrypted][Confidence: DPI][cat: Web/5][61 pkts/4478 bytes <-> 75 pkts/99815 bytes][Goodput ratio: 16/96][28.39 sec][bytes ratio: -0.914 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 600/30 23191/1117 3362/144][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 73/1331 534/1442 63/364][Risk: ** Self-signed Cert **** TLS (probably) Not Carrying HTTPS **** Missing SNI TLS Extn **** Malicious JA3 Fingerp. **][Risk Score: 210][TLSv1.2][JA3C: 51c64c77e60f3980eea90869b68c58a8][JA3S: ec74a5c51106f0419184d0dd08fb05bc][Issuer: C=GB, ST=London, L=London, O=Global Security, OU=IT Department, CN=example.com][Subject: C=GB, ST=London, L=London, O=Global Security, OU=IT Department, CN=example.com][Certificate SHA-1: 43:A2:39:73:AC:4D:2C:15:7B:D6:4E:32:EA:22:11:B7:97:65:1A:93][Firefox][Validity: 2022-04-21 10:08:46 - 2023-04-21 10:08:46][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,93,0,0,0,0]
6 TCP 10.4.25.101:49804 <-> 138.197.147.101:443 [proto: 91/TLS][Encrypted][Confidence: DPI][cat: Web/5][10 pkts/1517 bytes <-> 7 pkts/1208 bytes][Goodput ratio: 61/66][48.61 sec][bytes ratio: 0.113 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 5997/806 44782/3012 14692/1274][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 152/173 607/714 179/224][Risk: ** TLS (probably) Not Carrying HTTPS **** Missing SNI TLS Extn **** Malicious JA3 Fingerp. **][Risk Score: 110][TLSv1.2][JA3C: 51c64c77e60f3980eea90869b68c58a8][JA3S: fd4bc6cea4877646ccd62f0792ec0b62][Firefox][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 16,16,0,16,0,0,0,0,0,0,16,0,0,0,0,0,0,16,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 changes: 1 addition & 1 deletion tests/result/exe_download.pcap.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ Confidence DPI : 1 (flows)

HTTP 703 717463 1

1 TCP 10.9.25.101:49165 <-> 144.91.69.195:80 [proto: 7/HTTP][ClearText][Confidence: DPI][cat: Download/7][203 pkts/11127 bytes <-> 500 pkts/706336 bytes][Goodput ratio: 1/96][5.18 sec][Hostname/SNI: 144.91.69.195][bytes ratio: -0.969 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 23/9 319/365 49/37][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 55/1413 207/1514 11/134][URL: 144.91.69.195/solar.php][StatusCode: 200][Content-Type: application/octet-stream][User-Agent: pwtyyEKzNtGatwnJjmCcBLbOveCVpc][Risk: ** Binary App Transfer **** HTTP Numeric IP Address **][Risk Score: 260][PLAIN TEXT (GET /solar.php HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,2,0,0,7,0,0,63,0,0,24,0,0]
1 TCP 10.9.25.101:49165 <-> 144.91.69.195:80 [proto: 7/HTTP][ClearText][Confidence: DPI][cat: Download/7][203 pkts/11127 bytes <-> 500 pkts/706336 bytes][Goodput ratio: 1/96][5.18 sec][Hostname/SNI: 144.91.69.195][bytes ratio: -0.969 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 23/9 319/365 49/37][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 55/1413 207/1514 11/134][URL: 144.91.69.195/solar.php][StatusCode: 200][Content-Type: application/octet-stream][User-Agent: pwtyyEKzNtGatwnJjmCcBLbOveCVpc][Risk: ** Binary App Transfer **** HTTP Susp User-Agent **** HTTP Numeric IP Address **][Risk Score: 360][PLAIN TEXT (GET /solar.php HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,2,0,0,7,0,0,63,0,0,24,0,0]