Skip to content

Commit

Permalink
Improved suspicious http user agent detection. (#1537)
Browse files Browse the repository at this point in the history 
Signed-off-by: lns <matzeton@googlemail.com>
  • Loading branch information
@utoni
utoni authored May 2, 2022
1 parent c8f72ef commit 47d6a65
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 12 deletions.
47 changes: 36 additions & 11 deletions src/lib/protocols/http.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -420,19 +420,44 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp

static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
char *ua) {
u_int len;
char const *ua, size_t ua_len) {
char *double_slash;

if((!ua) || (ua[0] == '\0'))
return;
else
len = strlen(ua);

if (ua_len > 12)
{
size_t i, upper_case_count = 0;

for (i = 0; i < ua_len; ++i)
{
/*
* We assume at least one non alpha char.
* e.g. ' ', '-' or ';' ...
*/
if (isalpha(ua[i]) == 0)
{
break;
}
if (isupper(ua[i]) != 0)
{
upper_case_count++;
}
}

if (i == ua_len)
{
float upper_case_ratio = (float)upper_case_count / (float)ua_len;
if (upper_case_ratio >= 0.2f)
{
ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);
}
}
}

if((!strncmp(ua, "<?", 2))
|| strchr(ua, '$')
// || ndpi_check_dga_name(ndpi_struct, NULL, ua, 0)
// || ndpi_match_bigram(ndpi_struct, &ndpi_struct->impossible_bigrams_automa, ua)
)
ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);

Expand All @@ -448,8 +473,8 @@ static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_stru
if(!strncmp(ua, "jndi:ldap://", 12)) /* Log4J */ {
ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT);
} else if(
(len < 4) /* Too short */
|| (len > 256) /* Too long */
(ua_len < 4) /* Too short */
|| (ua_len > 256) /* Too long */
|| (!strncmp(ua, "test", 4))
|| strchr(ua, '{')
|| strchr(ua, '}')
Expand All @@ -464,8 +489,8 @@ static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_stru
*/
if((strstr(ua, "+http") != NULL)
|| (strstr(ua, " http") != NULL)
|| ndpi_strncasestr(ua, "Crawler", strlen(ua))
|| ndpi_strncasestr(ua, "Bot", strlen(ua)) /* bot/robot */
|| ndpi_strncasestr(ua, "Crawler", ua_len)
|| ndpi_strncasestr(ua, "Bot", ua_len) /* bot/robot */
) {
ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_CRAWLER_BOT);
}
Expand Down Expand Up @@ -537,7 +562,7 @@ int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,

if (ndpi_user_agent_set(flow, ua_ptr, ua_ptr_len) != NULL)
{
ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent);
ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent, ua_ptr_len);
} else {
NDPI_LOG_DBG2(ndpi_struct, "Could not set HTTP user agent\n");
}
Expand Down
Binary file added tests/pcap/emotet.pcap
View file
Binary file not shown.
20 changes: 20 additions & 0 deletions tests/result/emotet.pcap.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
Guessed flow protos: 1

DPI Packets (TCP): 121 (20.17 pkts/flow)
Confidence DPI : 6 (flows)

SMTP 626 438465 1
HTTP 1601 1581542 3
TLS 153 107018 2

JA3 Host Stats:
IP Address # JA3C
1 10.4.25.101 1


1 TCP 10.4.20.102:54319 <-> 107.161.178.210:80 [proto: 7/HTTP][ClearText][Confidence: DPI][cat: Web/5][272 pkts/16545 bytes <-> 557 pkts/800118 bytes][Goodput ratio: 1/96][9.12 sec][Hostname/SNI: gandhitoday.org][bytes ratio: -0.959 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 26/11 2171/1215 155/59][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 61/1436 279/1442 13/84][URL: gandhitoday.org/video/6JvA8/][StatusCode: 200][User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; Trident/7.0; rv:11.0) like Gecko][Risk: ** Binary App Transfer **][Risk Score: 250][PLAIN TEXT (GET /video/6J)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0]
2 TCP 10.4.25.101:49797 <-> 77.105.36.156:80 [proto: 7/HTTP][ClearText][Confidence: DPI][cat: Download/7][169 pkts/10292 bytes <-> 395 pkts/565664 bytes][Goodput ratio: 1/96][1.99 sec][Hostname/SNI: filmmogzivota.rs][bytes ratio: -0.964 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/4 292/171 38/19][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 61/1432 206/1442 11/107][URL: filmmogzivota.rs/SpryAssets/gDR/][StatusCode: 200][Content-Type: application/x-msdownload][User-Agent: vBKbaQgjyvRRbcgfvlsc][Risk: ** Binary App Transfer **** HTTP Susp User-Agent **][Risk Score: 350][PLAIN TEXT (GET /SpryAssets/gDR/ HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0]
3 TCP 10.2.25.102:57309 <-> 193.252.22.84:587 [proto: 3/SMTP][ClearText][Confidence: DPI][cat: Email/3][303 pkts/420177 bytes <-> 323 pkts/18288 bytes][Goodput ratio: 96/5][19.04 sec][Hostname/SNI: opmta1mto02nd1][bytes ratio: 0.917 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 26/66 1205/3211 138/351][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 1387/57 1514/214 400/13][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][PLAIN TEXT (220 opmta)][Plen Bins: 7,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,83,0,0]
4 TCP 10.3.29.101:56309 <-> 104.161.127.22:80 [proto: 7/HTTP][ClearText][Confidence: DPI][cat: Web/5][72 pkts/4883 bytes <-> 136 pkts/184040 bytes][Goodput ratio: 20/96][11.81 sec][Hostname/SNI: fkl.co.ke][bytes ratio: -0.948 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 224/98 7597/7597 1122/760][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 68/1353 591/1415 81/273][URL: fkl.co.ke/wp-content/Elw3kPvOsZxM5/][StatusCode: 200][Content-Type: text/html][User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.55][PLAIN TEXT (GET /wp)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0]
5 TCP 10.4.25.101:49803 <-> 138.197.147.101:443 [proto: 91/TLS][Encrypted][Confidence: DPI][cat: Web/5][61 pkts/4478 bytes <-> 75 pkts/99815 bytes][Goodput ratio: 16/96][28.39 sec][bytes ratio: -0.914 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 600/30 23191/1117 3362/144][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 73/1331 534/1442 63/364][Risk: ** Self-signed Cert **** TLS (probably) Not Carrying HTTPS **** Missing SNI TLS Extn **** Malicious JA3 Fingerp. **][Risk Score: 210][TLSv1.2][JA3C: 51c64c77e60f3980eea90869b68c58a8][JA3S: ec74a5c51106f0419184d0dd08fb05bc][Issuer: C=GB, ST=London, L=London, O=Global Security, OU=IT Department, CN=example.com][Subject: C=GB, ST=London, L=London, O=Global Security, OU=IT Department, CN=example.com][Certificate SHA-1: 43:A2:39:73:AC:4D:2C:15:7B:D6:4E:32:EA:22:11:B7:97:65:1A:93][Firefox][Validity: 2022-04-21 10:08:46 - 2023-04-21 10:08:46][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,93,0,0,0,0]
6 TCP 10.4.25.101:49804 <-> 138.197.147.101:443 [proto: 91/TLS][Encrypted][Confidence: DPI][cat: Web/5][10 pkts/1517 bytes <-> 7 pkts/1208 bytes][Goodput ratio: 61/66][48.61 sec][bytes ratio: 0.113 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 5997/806 44782/3012 14692/1274][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 152/173 607/714 179/224][Risk: ** TLS (probably) Not Carrying HTTPS **** Missing SNI TLS Extn **** Malicious JA3 Fingerp. **][Risk Score: 110][TLSv1.2][JA3C: 51c64c77e60f3980eea90869b68c58a8][JA3S: fd4bc6cea4877646ccd62f0792ec0b62][Firefox][Cipher: TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384][Plen Bins: 16,16,0,16,0,0,0,0,0,0,16,0,0,0,0,0,0,16,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 changes: 1 addition & 1 deletion tests/result/exe_download.pcap.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ Confidence DPI : 1 (flows)

HTTP 703 717463 1

1 TCP 10.9.25.101:49165 <-> 144.91.69.195:80 [proto: 7/HTTP][ClearText][Confidence: DPI][cat: Download/7][203 pkts/11127 bytes <-> 500 pkts/706336 bytes][Goodput ratio: 1/96][5.18 sec][Hostname/SNI: 144.91.69.195][bytes ratio: -0.969 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 23/9 319/365 49/37][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 55/1413 207/1514 11/134][URL: 144.91.69.195/solar.php][StatusCode: 200][Content-Type: application/octet-stream][User-Agent: pwtyyEKzNtGatwnJjmCcBLbOveCVpc][Risk: ** Binary App Transfer **** HTTP Numeric IP Address **][Risk Score: 260][PLAIN TEXT (GET /solar.php HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,2,0,0,7,0,0,63,0,0,24,0,0]
1 TCP 10.9.25.101:49165 <-> 144.91.69.195:80 [proto: 7/HTTP][ClearText][Confidence: DPI][cat: Download/7][203 pkts/11127 bytes <-> 500 pkts/706336 bytes][Goodput ratio: 1/96][5.18 sec][Hostname/SNI: 144.91.69.195][bytes ratio: -0.969 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 23/9 319/365 49/37][Pkt Len c2s/s2c min/avg/max/stddev: 54/54 55/1413 207/1514 11/134][URL: 144.91.69.195/solar.php][StatusCode: 200][Content-Type: application/octet-stream][User-Agent: pwtyyEKzNtGatwnJjmCcBLbOveCVpc][Risk: ** Binary App Transfer **** HTTP Susp User-Agent **** HTTP Numeric IP Address **][Risk Score: 360][PLAIN TEXT (GET /solar.php HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,2,0,0,7,0,0,63,0,0,24,0,0]

0 comments on commit 47d6a65

Please sign in to comment.