Added channel names shorter in the metrics/csv-timeline/json-timeline command

CHANGELOG-Japanese.md

@@ -10,6 +10,7 @@
 
 - オプションのグループ分けを再修正した。(#918)(@hitenkoku)
 - JSONL形式のログを読み込む際のメモリ使用量を約75%削減した。 (#921) (@fukusuket)
+- `rules/config/channel_abbreviations_generic.txt`によってチャンネル名の一般的な単語名を省略する機能をmetrics、json-timeline、csv-timelineに追加した。 (#923) (@hitenkoku)
 
 **バグ修正:**
 

CHANGELOG.md

@@ -10,6 +10,7 @@
 
 - Reorganized the grouping of command line options. (#918) (@hitenkoku)
 - Reduced memory usage by approximately 75% when reading JSONL formatted logs. (#921) (@fukusuket)
+- Channel names are now further abbreviated in the metrics, json-timeline, csv-timeline commands according to `rules/config/channel_abbreviations_generic.txt`. (#923) (@hitenkoku)
 
 **Bug Fixes:**
 

Cargo.toml

@@ -48,6 +48,7 @@ ureq = "*"
 mockall = "*"
 maxminddb = "0.*"
 cidr-utils = "0.*"
+aho-corasick = "*"
 
 [profile.dev]
 debug = 0

src/afterfact.rs

@@ -1518,8 +1518,10 @@ mod tests {
 
     #[test]
     fn test_emit_csv_output() {
-        let mock_ch_filter =
-            message::create_output_filter_config("test_files/config/channel_abbreviations.txt");
+        let mock_ch_filter = message::create_output_filter_config(
+            "test_files/config/channel_abbreviations.txt",
+            true,
+        );
         let test_filepath: &str = "test.evtx";
         let test_rulepath: &str = "test-rule.yml";
         let test_title = "test_title";

src/detections/configs.rs

@@ -4,10 +4,12 @@ use crate::detections::utils;
 use crate::options::geoip_search::GeoIPSearch;
 use crate::options::htmlreport;
 use crate::options::profile::{load_profile, Profile};
+use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind};
 use chrono::{DateTime, Utc};
 use clap::{ArgGroup, Args, ColorChoice, Command, CommandFactory, Parser, Subcommand};
 use compact_str::CompactString;
 use hashbrown::{HashMap, HashSet};
+use itertools::Itertools;
 use lazy_static::lazy_static;
 use nested::Nested;
 use regex::Regex;
@@ -50,6 +52,8 @@ pub struct StoredStatic {
     pub config_path: PathBuf,
     pub eventkey_alias: EventKeyAliasConfig,
     pub ch_config: HashMap<CompactString, CompactString>,
+    pub ch_disp_abbr_generic: AhoCorasick,
+    pub ch_disp_abbr_gen_rep_values: Vec<CompactString>,
     pub quiet_errors_flag: bool,
     pub verbose_flag: bool,
     pub metrics_flag: bool,
@@ -258,6 +262,20 @@ impl StoredStatic {
             Some(Action::LogonSummary(opt)) => opt.output.as_ref(),
             _ => None,
         };
+        let general_ch_abbr = create_output_filter_config(
+            utils::check_setting_path(config_path, "channel_abbreviations_generic.txt", false)
+                .unwrap_or_else(|| {
+                    utils::check_setting_path(
+                        &CURRENT_EXE_PATH.to_path_buf(),
+                        "rules/config/channel_abbreviations_generic.txt",
+                        true,
+                    )
+                    .unwrap()
+                })
+                .to_str()
+                .unwrap(),
+            false,
+        );
         let mut ret = StoredStatic {
             config: input_config.as_ref().unwrap().to_owned(),
             config_path: config_path.to_path_buf(),
@@ -273,7 +291,15 @@ impl StoredStatic {
                     })
                     .to_str()
                     .unwrap(),
+                true,
             ),
+            ch_disp_abbr_generic: AhoCorasickBuilder::new()
+                .match_kind(MatchKind::LeftmostLongest)
+                .build(general_ch_abbr.keys().map(|x| x.as_str())),
+            ch_disp_abbr_gen_rep_values: general_ch_abbr
+                .values()
+                .map(|x| x.to_owned())
+                .collect_vec(),
             default_details: Self::get_default_details(
                 utils::check_setting_path(config_path, "default_details.txt", false)
                     .unwrap_or_else(|| {

src/detections/detection.rs

@@ -284,10 +284,16 @@ impl Detection {
                         key.as_str(),
                         Channel(
                             stored_static
-                                .ch_config
-                                .get(&CompactString::from(ch_str.to_ascii_lowercase()))
-                                .unwrap_or(ch_str)
-                                .to_owned(),
+                                .ch_disp_abbr_generic
+                                .replace_all(
+                                    stored_static
+                                        .ch_config
+                                        .get(&CompactString::from(ch_str.to_ascii_lowercase()))
+                                        .unwrap_or(ch_str)
+                                        .as_str(),
+                                    &stored_static.ch_disp_abbr_gen_rep_values,
+                                )
+                                .into(),
                         ),
                     );
                 }

src/detections/message.rs

@@ -48,6 +48,7 @@ lazy_static! {
         utils::check_setting_path(&CURRENT_EXE_PATH.to_path_buf(), "config/mitre_tactics.txt", true)
             .unwrap().to_str()
             .unwrap(),
+        true
     );
     pub static ref LEVEL_ABBR_MAP:HashMap<&'static str, &'static str> = HashMap::from_iter(vec![
         ("critical", "crit"),
@@ -68,7 +69,10 @@ lazy_static! {
 
 /// ファイルパスで記載されたtagでのフル名、表示の際に置き換えられる文字列のHashMapを作成する関数。
 /// ex. attack.impact,Impact
-pub fn create_output_filter_config(path: &str) -> HashMap<CompactString, CompactString> {
+pub fn create_output_filter_config(
+    path: &str,
+    is_lower_case: bool,
+) -> HashMap<CompactString, CompactString> {
     let mut ret: HashMap<CompactString, CompactString> = HashMap::new();
     let read_result = utils::read_csv(path);
     if read_result.is_err() {
@@ -80,8 +84,13 @@ pub fn create_output_filter_config(path: &str) -> HashMap<CompactString, Compact
             return;
         }
 
+        let key = if is_lower_case {
+            line[0].trim().to_ascii_lowercase()
+        } else {
+            line[0].trim().to_string()
+        };
         ret.insert(
-            CompactString::from(line[0].trim().to_ascii_lowercase()),
+            CompactString::from(key),
             CompactString::from(line[1].trim()),
         );
     });
@@ -607,7 +616,7 @@ mod tests {
     #[test]
     /// test of loading output filter config by mitre_tactics.txt
     fn test_load_mitre_tactics_log() {
-        let actual = create_output_filter_config("test_files/config/mitre_tactics.txt");
+        let actual = create_output_filter_config("test_files/config/mitre_tactics.txt", true);
         let expected: HashMap<CompactString, CompactString> = HashMap::from([
             ("attack.impact".into(), "Impact".into()),
             ("xxx".into(), "yyy".into()),
@@ -618,8 +627,10 @@ mod tests {
     #[test]
     /// loading test to channel_abbrevations.txt
     fn test_load_abbrevations() {
-        let actual = create_output_filter_config("test_files/config/channel_abbreviations.txt");
-        let actual2 = create_output_filter_config("test_files/config/channel_abbreviations.txt");
+        let actual =
+            create_output_filter_config("test_files/config/channel_abbreviations.txt", true);
+        let actual2 =
+            create_output_filter_config("test_files/config/channel_abbreviations.txt", true);
         let expected: HashMap<CompactString, CompactString> = HashMap::from([
             ("security".into(), "Sec".into()),
             ("xxx".into(), "yyy".into()),

src/main.rs

@@ -1105,12 +1105,7 @@ impl App {
             ));
 
             // timeline機能の実行
-            tl.start(
-                &records_per_detect,
-                stored_static.metrics_flag,
-                stored_static.logon_summary_flag,
-                &stored_static.eventkey_alias,
-            );
+            tl.start(&records_per_detect, stored_static);
 
             if !(stored_static.metrics_flag || stored_static.logon_summary_flag) {
                 // ruleファイルの検知
@@ -1251,12 +1246,7 @@ impl App {
             ));
 
             // timeline機能の実行
-            tl.start(
-                &records_per_detect,
-                stored_static.metrics_flag,
-                stored_static.logon_summary_flag,
-                &stored_static.eventkey_alias,
-            );
+            tl.start(&records_per_detect, stored_static);
 
             if !(stored_static.metrics_flag || stored_static.logon_summary_flag) {
                 // ruleファイルの検知

src/timeline/metrics.rs

@@ -1,4 +1,8 @@
-use crate::detections::{configs::EventKeyAliasConfig, detection::EvtxRecordInfo, utils};
+use crate::detections::{
+    configs::{EventKeyAliasConfig, StoredStatic},
+    detection::EvtxRecordInfo,
+    utils,
+};
 use compact_str::CompactString;
 use hashbrown::HashMap;
 
@@ -51,22 +55,17 @@ impl EventMetrics {
         }
     }
 
-    pub fn evt_stats_start(
-        &mut self,
-        records: &[EvtxRecordInfo],
-        metrics_flag: bool,
-        eventkey_alias: &EventKeyAliasConfig,
-    ) {
+    pub fn evt_stats_start(&mut self, records: &[EvtxRecordInfo], stored_static: &StoredStatic) {
         // 引数でmetricsオプションが指定されている時だけ、統計情報を出力する。
-        if !metrics_flag {
+        if !stored_static.metrics_flag {
             return;
         }
 
         // _recordsから、EventIDを取り出す。
-        self.stats_time_cnt(records, eventkey_alias);
+        self.stats_time_cnt(records, &stored_static.eventkey_alias);
 
         // EventIDで集計
-        self.stats_eventid(records, eventkey_alias);
+        self.stats_eventid(records, stored_static);
     }
 
     pub fn logon_stats_start(
@@ -111,18 +110,28 @@ impl EventMetrics {
         self.total += records.len();
     }
 
-    /// EventID`で集計
-    fn stats_eventid(&mut self, records: &[EvtxRecordInfo], eventkey_alias: &EventKeyAliasConfig) {
-        //        let mut evtstat_map = HashMap::new();
+    /// EventIDで集計
+    fn stats_eventid(&mut self, records: &[EvtxRecordInfo], stored_static: &StoredStatic) {
         for record in records.iter() {
             let channel = if let Some(ch) =
-                utils::get_event_value("Channel", &record.record, eventkey_alias)
+                utils::get_event_value("Channel", &record.record, &stored_static.eventkey_alias)
             {
-                ch.to_string()
+                stored_static.ch_disp_abbr_generic.replace_all(
+                    stored_static
+                        .ch_config
+                        .get(&CompactString::from(
+                            ch.to_string().replace('\"', "").to_ascii_lowercase(),
+                        ))
+                        .unwrap_or(&CompactString::from(ch.to_string().replace('\"', "")))
+                        .as_str(),
+                    &stored_static.ch_disp_abbr_gen_rep_values,
+                )
             } else {
                 "-".to_string()
             };
-            if let Some(idnum) = utils::get_event_value("EventID", &record.record, eventkey_alias) {
+            if let Some(idnum) =
+                utils::get_event_value("EventID", &record.record, &stored_static.eventkey_alias)
+            {
                 let count: &mut usize = self
                     .stats_list
                     .entry((idnum.to_string().replace('\"', ""), channel))

src/timeline/timelines.rs

@@ -2,7 +2,7 @@ use std::fs::File;
 use std::io::BufWriter;
 use std::path::PathBuf;
 
-use crate::detections::configs::{Action, EventInfoConfig, EventKeyAliasConfig, StoredStatic};
+use crate::detections::configs::{Action, EventInfoConfig, StoredStatic};
 use crate::detections::detection::EvtxRecordInfo;
 use crate::detections::message::AlertMessage;
 use comfy_table::modifiers::UTF8_ROUND_CORNERS;
@@ -40,17 +40,13 @@ impl Timeline {
         Timeline { stats: statistic }
     }
 
-    pub fn start(
-        &mut self,
-        records: &[EvtxRecordInfo],
-        metrics_flag: bool,
-        logon_summary_flag: bool,
-        eventkey_alias: &EventKeyAliasConfig,
-    ) {
-        self.stats
-            .evt_stats_start(records, metrics_flag, eventkey_alias);
-        self.stats
-            .logon_stats_start(records, logon_summary_flag, eventkey_alias);
+    pub fn start(&mut self, records: &[EvtxRecordInfo], stored_static: &StoredStatic) {
+        self.stats.evt_stats_start(records, stored_static);
+        self.stats.logon_stats_start(
+            records,
+            stored_static.logon_summary_flag,
+            &stored_static.eventkey_alias,
+        );
     }
 
     pub fn tm_stats_dsp_msg(
@@ -174,7 +170,16 @@ impl Timeline {
                 .is_some();
             // event_id_info.txtに登録あるものは情報設定
             // 出力メッセージ1行作成
-            let ch = channel_config
+            let ch = stored_static.ch_disp_abbr_generic.replace_all(
+                stored_static
+                    .ch_config
+                    .get(fmted_channel.to_lowercase().as_str())
+                    .unwrap_or(&fmted_channel)
+                    .as_str(),
+                &stored_static.ch_disp_abbr_gen_rep_values,
+            );
+
+            channel_config
                 .get(fmted_channel.to_lowercase().as_str())
                 .unwrap_or(&fmted_channel)
                 .to_string();