Fix JA4SSH and JA4H (#69)

* [fix] Generate a JA4SSH fingerprint every 200 _SSH_ packets * Fix calculation of JA4H_c Related issue: #58
FoxIO-LLC · Feb 4, 2024 · db49b59 · db49b59
1 parent fc3a3b3
commit db49b59
Show file tree

Hide file tree

Showing 16 changed files with 221 additions and 80 deletions.
diff --git a/pcap/badcurveball.pcap b/pcap/badcurveball.pcap
diff --git a/pcap/single-packets.pcap b/pcap/single-packets.pcap
diff --git a/pcap/tls-alpn-h2.pcap b/pcap/tls-alpn-h2.pcap
diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md
@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.18.0] - 2024-02-04
+
+### Fixed
+
+- Generate a JA4SSH fingerprint every 200 *SSH* (layer 7) packets.
+- Fix calculation of JA4H\_c (#58).
+
 ## [0.17.0] - 2024-01-31
 
 ### Fixed
@@ -70,7 +77,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add Rust sources of `ja4` and `ja4x` CLI tools.
 
-[unreleased]: https://github.com/FoxIO-LLC/ja4/compare/v0.17.0...HEAD
+[unreleased]: https://github.com/FoxIO-LLC/ja4/compare/v0.18.0...HEAD
+[0.18.0]: https://github.com/FoxIO-LLC/ja4/compare/v0.17.0...v0.18.0
 [0.17.0]: https://github.com/FoxIO-LLC/ja4/compare/v0.16.2...v0.17.0
 [0.16.2]: https://github.com/FoxIO-LLC/ja4/compare/v0.16.1...v0.16.2
 [0.16.1]: https://github.com/FoxIO-LLC/ja4/compare/v0.16.0...v0.16.1

diff --git a/rust/Cargo.lock b/rust/Cargo.lock
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
@@ -3,7 +3,7 @@ members = ["ja4", "ja4x"]
 resolver = "2"
 
 [workspace.package]
-version = "0.17.0"
+version = "0.18.0"
 license = "LicenseRef-FoxIO-Proprietary"
 repository = "https://github.com/FoxIO-LLC/ja4"
 

diff --git a/rust/ja4/config.toml b/rust/ja4/config.toml
@@ -7,7 +7,7 @@
 [ssh]
 # enabled = true
 
-## How many packets to collect per SSH TCP stream before generating a JA4SSH fingerprint
+## New JA4SSH fingerprint is generated every `sample_size` SSH packets
 # sample_size = 200
 
 

diff --git a/rust/ja4/src/conf.rs b/rust/ja4/src/conf.rs
@@ -28,11 +28,11 @@ pub(crate) struct ConfSsh {
     pub(crate) enabled: bool,
     /// JA4SSH (SSH traffic fingerprinting) runs every `sample_size` packets
     /// per SSH TCP stream.
-    pub(crate) sample_size: u32,
+    pub(crate) sample_size: usize,
 }
 
 impl ConfSsh {
-    const DEFAULT_SAMPLE_SIZE: u32 = 200;
+    const DEFAULT_SAMPLE_SIZE: usize = 200;
 
     fn prepare(mut self) -> Self {
         if self.enabled && self.sample_size == 0 {

diff --git a/rust/ja4/src/http.rs b/rust/ja4/src/http.rs
@@ -187,15 +187,19 @@ impl HttpStats {
         let first_chunk =
             format!("{req_method}{version}{cookie_marker}{referer_marker}{nr_headers:02}{lang}");
 
+        let mut cookie_names = cookie_names(&cookies).collect_vec();
+
         if !original_order {
+            cookie_names.sort_unstable();
             cookies.sort_unstable();
         }
 
         let headers = headers.into_iter().join(",");
-        let (cookie_keys, cookie_items) = cookie_keys_and_items(&cookies);
+        let cookie_names = cookie_names.into_iter().join(",");
+        let cookies = cookies.into_iter().join(",");
 
         let ja4h_r = with_raw.then(|| {
-            let s = format!("{first_chunk}_{headers}_{cookie_keys}_{cookie_items}");
+            let s = format!("{first_chunk}_{headers}_{cookie_names}_{cookies}");
             if original_order {
                 Ja4hRawFingerprint::Unsorted(s)
             } else {
@@ -204,10 +208,10 @@ impl HttpStats {
         });
 
         let headers = crate::hash12(headers);
-        let cookie_keys = crate::hash12(cookie_keys);
-        let cookie_items = crate::hash12(cookie_items);
+        let cookie_names = crate::hash12(cookie_names);
+        let cookies = crate::hash12(cookies);
         let ja4h = {
-            let s = format!("{first_chunk}_{headers}_{cookie_keys}_{cookie_items}");
+            let s = format!("{first_chunk}_{headers}_{cookie_names}_{cookies}");
             if original_order {
                 Ja4hFingerprint::Unsorted(s)
             } else {
@@ -223,47 +227,36 @@ impl HttpStats {
     }
 }
 
-/// Returns a comma-separated list of cookie keys ("key1,key2,key3") and a comma-separated
-/// list of cookies ("key1=value1,key2=value2,key3=value3") .
-fn cookie_keys_and_items<S: AsRef<str>>(cookies: &[S]) -> (String, String) {
-    cookies
-        .iter()
-        .fold((String::new(), String::new()), |(keys, items), cookie| {
-            let cookie = cookie.as_ref();
-            // SAFETY: `split` never returns an empty iterator, so it's safe to unwrap.
-            let key = cookie.split('=').next().unwrap();
-
-            let keys = if keys.is_empty() {
-                key.to_owned()
-            } else {
-                format!("{keys},{key}")
-            };
-
-            let items = if items.is_empty() {
-                cookie.to_owned()
-            } else {
-                format!("{items},{cookie}")
-            };
-
-            (keys, items)
-        })
+/// Returns an iterator of owned cookie names.
+fn cookie_names<S: AsRef<str>>(cookies: &[S]) -> impl Iterator<Item = String> + '_ {
+    cookies.iter().map(|cookie| {
+        // SAFETY: `split` never returns an empty iterator, so it's safe to unwrap.
+        cookie.as_ref().split('=').next().unwrap().to_owned()
+    })
 }
 
 #[test]
-fn test_cookie_keys_and_items() {
+fn test_cookie_names() {
+    let no_cookies: [&str; 0] = [];
+    assert!(cookie_names(&no_cookies).next().is_none());
+
     assert_eq!(
-        cookie_keys_and_items(&["foo=bar", "baz=qux"]),
-        ("foo,baz".to_owned(), "foo=bar,baz=qux".to_owned())
+        cookie_names(&["foo=bar", "baz=qux"]).collect::<Vec<_>>(),
+        ["foo", "baz"]
     );
+
     assert_eq!(
-        cookie_keys_and_items(&["a=5", "c=3", "b=2", "a=1", "a=4"]),
-        ("a,c,b,a,a".to_owned(), "a=5,c=3,b=2,a=1,a=4".to_owned())
+        cookie_names(&["a=5", "c=3=4=5", "b=2", "a=1", "a=4"]).collect::<Vec<_>>(),
+        ["a", "c", "b", "a", "a"]
     );
 
-    let no_cookies: [&str; 0] = [];
     assert_eq!(
-        cookie_keys_and_items(&no_cookies),
-        (String::new(), String::new())
+        cookie_names(&[
+            "pardot=tee2foreb3fefpgvk8u1056vt3",
+            "visitor_id413862-hash=1f00bdb076b5fb707c70254849819ec1797d3e27cef91a61a9488cb7ca0ebf77f226caa4075591b2591bf9a1ccdf29432c67379b",
+            "visitor_id413862=286585660",
+        ]).collect_vec(),
+        ["pardot", "visitor_id413862-hash", "visitor_id413862"]
     );
 }
 

diff --git a/rust/ja4/src/snapshots/ja4__insta@badcurveball.pcap.snap b/rust/ja4/src/snapshots/ja4__insta@badcurveball.pcap.snap
@@ -0,0 +1,35 @@
+---
+source: ja4/src/lib.rs
+expression: output
+---
+- stream: 0
+  transport: tcp
+  src: 172.130.128.76
+  dst: 54.226.182.138
+  src_port: 55318
+  dst_port: 443
+  tls_server_name: bad.curveballtest.com
+  ja4: t13d1615h2_46e7e9700bed_45f260be83e2
+  ja4s: t1205h1_c02b_845f7282a956
+  tls_certs:
+  - x509:
+    - ja4x: 2e9214a636bc_a373a9f83c6b_0e17604154c5
+      issuerCountryName: HR
+      issuerStateOrProvinceName: Zagreb
+      issuerOrganizationName: INFIGO IS
+      issuerCommonName: INFIGO
+      subjectCountryName: US
+      subjectOrganizationName: SANS Internet Storm Center
+      subjectCommonName: SANS ISC DShield Test
+    - ja4x: 2e9214a636bc_2e9214a636bc_795797892f9c
+      issuerCountryName: HR
+      issuerStateOrProvinceName: Zagreb
+      issuerOrganizationName: INFIGO IS
+      issuerCommonName: INFIGO
+      subjectCountryName: HR
+      subjectStateOrProvinceName: Zagreb
+      subjectOrganizationName: INFIGO IS
+      subjectCommonName: INFIGO
+  ja4l_c: 2177_64
+  ja4l_s: 781_238
+
diff --git a/rust/ja4/src/snapshots/ja4__insta@single-packets.pcap.snap b/rust/ja4/src/snapshots/ja4__insta@single-packets.pcap.snap
@@ -0,0 +1,69 @@
+---
+source: ja4/src/lib.rs
+expression: output
+---
+- stream: 0
+  transport: tcp
+  src: 192.168.25.150
+  dst: 74.125.24.149
+  src_port: 49677
+  dst_port: 80
+  http:
+  - ja4h: ge11cr06enus_8c2f9ef95269_2a79f5d9f8b3_7b4d78c057bc
+- stream: 1
+  transport: tcp
+  src: 192.168.25.150
+  dst: 118.215.80.242
+  src_port: 49654
+  dst_port: 80
+  http:
+  - ja4h: ge11cr07enus_45c71a3fb6ea_a25bf252eb59_43a9e3e95c85
+- stream: 2
+  transport: tcp
+  src: 192.168.25.150
+  dst: 13.115.50.210
+  src_port: 49683
+  dst_port: 80
+  http:
+  - ja4h: ge11nr06enus_8c2f9ef95269_000000000000_000000000000
+- stream: 3
+  transport: tcp
+  src: 192.168.25.150
+  dst: 104.89.119.175
+  src_port: 49708
+  dst_port: 80
+  http:
+  - ja4h: po11cr09enus_130d8cd1913c_f81c0e5c6793_90689f748de6
+- stream: 4
+  transport: tcp
+  src: 192.168.25.150
+  dst: 193.242.192.43
+  src_port: 49735
+  dst_port: 80
+  http:
+  - ja4h: ge11cr07enus_45c71a3fb6ea_9ee64e91aa30_109254663367
+- stream: 5
+  transport: tcp
+  src: 192.168.25.150
+  dst: 74.125.24.100
+  src_port: 49733
+  dst_port: 80
+  http:
+  - ja4h: ge11nr06enus_8c2f9ef95269_000000000000_000000000000
+- stream: 6
+  transport: tcp
+  src: 192.168.25.150
+  dst: 74.125.24.95
+  src_port: 49743
+  dst_port: 80
+  http:
+  - ja4h: ge11nr06enus_8c2f9ef95269_000000000000_000000000000
+- stream: 7
+  transport: tcp
+  src: 192.168.25.150
+  dst: 35.174.150.168
+  src_port: 49738
+  dst_port: 80
+  http:
+  - ja4h: ge11cr06enus_8c2f9ef95269_d23bf79698dc_c1eaa758c543
+
diff --git a/rust/ja4/src/snapshots/ja4__insta@ssh-r.pcap.snap b/rust/ja4/src/snapshots/ja4__insta@ssh-r.pcap.snap
@@ -11,9 +11,8 @@ expression: output
   ja4l_c: 94_128
   ja4l_s: 32_64
   ja4ssh:
-  - c64s64_c78s65_c45s10
-  - c64s64_c59s72_c67s2
-  - c64s64_c3s4_c4s0
+  - c64s64_c107s93_c74s10
+  - c64s64_c33s48_c42s2
   ssh_extras:
     hassh: e77c2db7432e8cfbc42a96909a84fc8e
     hassh_server: 6832f1ce43d4397c2c0a3e2f8c94334e
@@ -45,14 +44,11 @@ expression: output
   ja4l_c: 12_64
   ja4l_s: 3169_116
   ja4ssh:
-  - c76s76_c63s69_c19s47
-  - c76s76_c74s57_c0s69
-  - c76s76_c71s60_c0s69
-  - c76s76_c69s62_c0s69
-  - c76s76_c71s59_c0s70
-  - c76s76_c75s57_c0s68
-  - c76s76_c70s69_c6s55
-  - c36s60_c3s3_c4s1
+  - c76s76_c104s96_c19s82
+  - c76s76_c108s92_c0s105
+  - c76s76_c106s94_c0s107
+  - c76s76_c111s89_c0s102
+  - c76s76_c67s65_c10s52
   ssh_extras:
     hassh: ec9ea89c70f5fc71cf61061bff5e4740
     hassh_server: 2307c390c7c9aba5b4c9519e72347f34

diff --git a/rust/ja4/src/snapshots/ja4__insta@ssh-scp-1050.pcap.snap b/rust/ja4/src/snapshots/ja4__insta@ssh-scp-1050.pcap.snap
@@ -11,11 +11,11 @@ expression: output
   ja4l_c: 179_128
   ja4l_s: 38_64
   ja4ssh:
-  - c112s80_c52s107_c35s4
-  - c0s1460_c0s174_c26s0
-  - c112s1460_c13s150_c37s0
-  - c0s1460_c0s178_c22s0
-  - c0s1460_c0s179_c21s0
+  - c112s1460_c52s148_c41s4
+  - c112s1460_c13s187_c35s0
+  - c0s1460_c0s200_c36s0
+  - c0s1460_c0s200_c23s0
+  - c0s1460_c0s53_c6s0
   ssh_extras:
     hassh: eb6d4c713c7dcaba7cfd070b095213a9
     hassh_server: 6832f1ce43d4397c2c0a3e2f8c94334e

diff --git a/rust/ja4/src/snapshots/ja4__insta@ssh2.pcapng.snap b/rust/ja4/src/snapshots/ja4__insta@ssh2.pcapng.snap
@@ -172,9 +172,8 @@ expression: output
   ja4l_c: 77_128
   ja4l_s: 12897_50
   ja4ssh:
-  - c36s36_c55s87_c51s5
-  - c36s36_c49s90_c59s2
-  - c36s36_c14s23_c15s0
+  - c36s36_c76s124_c74s5
+  - c36s52_c42s76_c51s2
   ssh_extras:
     hassh: 06046964c022c6407d15a27b12a6a4fb
     hassh_server: 699519fdcc30cbcd093d5cd01e4b1d56

diff --git a/rust/ja4/src/snapshots/ja4__insta@tls-alpn-h2.pcap.snap b/rust/ja4/src/snapshots/ja4__insta@tls-alpn-h2.pcap.snap
@@ -0,0 +1,41 @@
+---
+source: ja4/src/lib.rs
+expression: output
+---
+- stream: 0
+  transport: tcp
+  src: 2001:4998:ef83:14:8000::100d
+  dst: 2606:4700::6811:d209
+  src_port: 64034
+  dst_port: 443
+  tls_server_name: www.cloudflare.com
+  ja4: t12d4605h2_85626a9a5f7f_aaf95bb78ec9
+  ja4s: t1204h2_cca9_1428ce7b4018
+  tls_certs:
+  - x509:
+    - ja4x: 7d5dbb3783b4_ba7ce0880c07_7bf9a7bf7029
+      issuerCountryName: US
+      issuerOrganizationName: DigiCert Inc
+      issuerOrganizationalUnit: www.digicert.com
+      issuerCommonName: DigiCert ECC Extended Validation Server CA
+      subjectBusinessCategory: Private Organization
+      subjectMsJurisdictionCountry: US
+      subjectMsJurisdictionStateOrProvince: Delaware
+      subjectSerialNumber: '4710875'
+      subjectCountryName: US
+      subjectStateOrProvinceName: California
+      subjectLocalityName: San Francisco
+      subjectOrganizationName: Cloudflare, Inc.
+      subjectCommonName: cloudflare.com
+    - ja4x: 7d5dbb3783b4_7d5dbb3783b4_41a019652939
+      issuerCountryName: US
+      issuerOrganizationName: DigiCert Inc
+      issuerOrganizationalUnit: www.digicert.com
+      issuerCommonName: DigiCert High Assurance EV Root CA
+      subjectCountryName: US
+      subjectOrganizationName: DigiCert Inc
+      subjectOrganizationalUnit: www.digicert.com
+      subjectCommonName: DigiCert ECC Extended Validation Server CA
+  ja4l_c: 35_64
+  ja4l_s: 18861_59
+