Add statistics tracking

.rustfmt.toml

@@ -0,0 +1 @@
+reorder_modules = false

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.11.0"
+version = "1.11.1"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -30,15 +30,15 @@ reqwest = { version = "0.10", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
-serde = { version = "1.0", features = ["derive"] }
+serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
-console = "0.13"
+console = "0.14"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
-crossterm = "0.18"
+crossterm = "0.19"
 rlimit = "0.5"
 ctrlc = "3.1"
 fuzzyhash = "0.2"

README.md

@@ -78,6 +78,10 @@ Enumeration.
     - [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level)
     - [ferox-config.toml](#ferox-configtoml)
     - [Command Line Parsing](#command-line-parsing)
+- [Scan's Display Explained](#-scans-display-explained)
+    - [Discovered Resource](#discovered-resource)
+    - [Overall Scan Progress Bar](#overall-scan-progress-bar)
+    - [Directory Scan Progress Bar](#directory-scan-progress-bar)
 - [Example Usage](#-example-usage)
     - [Multiple Values](#multiple-values)
     - [Include Headers](#include-headers)
@@ -460,6 +464,31 @@ OPTIONS:
     -w, --wordlist <FILE>                   Path to the wordlist
 ```
 
+## 📊 Scan's Display Explained
+
+`feroxbuster` attempts to be intuitive and easy to understand, however, if you are wondering about any of the scan's
+output and what it means, this is the section for you!  
+
+### Discovered Resource
+
+When `feroxbuster` finds a response that you haven't filtered out, it's reported above the progress bars and looks similar to what's pictured below.  
+
+The number of lines, words, and bytes shown here can be used to [filter those responses](#filter-response-by-word-count--line-count--new-in-v160)
+
+![response-bar-explained](img/response-bar-explained.png)
+
+### Overall Scan Progress Bar
+
+The top progress bar, colored yellow, tracks the overall scan status.  Its fields are described in the image below.
+
+![total-bar-explained](img/total-bar-explained.png)
+
+### Directory Scan Progress Bar
+
+All other progress bars, colored cyan, represent a scan of one particular directory and will look similar to what's below.   
+
+![dir-scan-bar-explained](img/dir-scan-bar-explained.png)
+
 ## 🧰 Example Usage
 
 ### Multiple Values

src/banner.rs

@@ -1,9 +1,13 @@
-use crate::config::{Configuration, CONFIGURATION};
-use crate::utils::{make_request, status_colorizer};
+use crate::{
+    config::{Configuration, CONFIGURATION},
+    statistics::StatCommand,
+    utils::{make_request, status_colorizer},
+};
 use console::{style, Emoji};
 use reqwest::{Client, Url};
 use serde_json::Value;
 use std::io::Write;
+use tokio::sync::mpsc::UnboundedSender;
 
 /// macro helper to abstract away repetitive string formatting
 macro_rules! format_banner_entry_helper {
@@ -67,8 +71,13 @@ enum UpdateStatus {
 /// ex: v1.1.0
 ///
 /// Returns `UpdateStatus`
-async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateStatus {
-    log::trace!("enter: needs_update({:?}, {})", client, url);
+async fn needs_update(
+    client: &Client,
+    url: &str,
+    bin_version: &str,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> UpdateStatus {
+    log::trace!("enter: needs_update({:?}, {}, {:?})", client, url, tx_stats);
 
     let unknown = UpdateStatus::Unknown;
 
@@ -81,7 +90,7 @@ async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateSt
         }
     };
 
-    if let Ok(response) = make_request(&client, &api_url).await {
+    if let Ok(response) = make_request(&client, &api_url, tx_stats.clone()).await {
         let body = response.text().await.unwrap_or_default();
 
         let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
@@ -137,8 +146,13 @@ fn format_emoji(emoji: &str) -> String {
 /// Prints the banner to stdout.
 ///
 /// Only prints those settings which are either always present, or passed in by the user.
-pub async fn initialize<W>(targets: &[String], config: &Configuration, version: &str, mut writer: W)
-where
+pub async fn initialize<W>(
+    targets: &[String],
+    config: &Configuration,
+    version: &str,
+    mut writer: W,
+    tx_stats: UnboundedSender<StatCommand>,
+) where
     W: Write,
 {
     let artwork = format!(
@@ -150,7 +164,7 @@ by Ben "epi" Risher {}                 ver: {}"#,
         Emoji("🤓", &format!("{:<2}", "\u{0020}")),
         version
     );
-    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
+    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version, tx_stats).await;
 
     let top = "───────────────────────────┬──────────────────────";
     let addl_section = "──────────────────────────────────────────────────";
@@ -536,59 +550,80 @@ by Ben "epi" Risher {}                 ver: {}"#,
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::VERSION;
+    use crate::{FeroxChannel, VERSION};
     use httpmock::Method::GET;
     use httpmock::MockServer;
     use std::fs::read_to_string;
     use std::io::stderr;
     use std::time::Duration;
     use tempfile::NamedTempFile;
+    use tokio::sync::mpsc;
 
     #[tokio::test(core_threads = 1)]
     /// test to hit no execution of targets for loop in banner
     async fn banner_intialize_without_targets() {
         let config = Configuration::default();
-        initialize(&[], &config, VERSION, stderr()).await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        initialize(&[], &config, VERSION, stderr(), tx).await;
     }
 
     #[tokio::test(core_threads = 1)]
     /// test to hit no execution of statuscode for loop in banner
     async fn banner_intialize_without_status_codes() {
-        let mut config = Configuration::default();
-        config.status_codes = vec![];
+        let config = Configuration {
+            status_codes: vec![],
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
 
     #[tokio::test(core_threads = 1)]
     /// test to hit an empty config file
     async fn banner_intialize_without_config_file() {
-        let mut config = Configuration::default();
-        config.config = String::new();
+        let config = Configuration {
+            config: String::new(),
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
 
     #[tokio::test(core_threads = 1)]
     /// test to hit an empty config file
     async fn banner_intialize_without_queries() {
-        let mut config = Configuration::default();
-        config.queries = vec![(String::new(), String::new())];
+        let config = Configuration {
+            queries: vec![(String::new(), String::new())],
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
@@ -598,11 +633,14 @@ mod tests {
     async fn banner_intialize_with_mismatched_version() {
         let config = Configuration::default();
         let file = NamedTempFile::new().unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             "mismatched-version",
             &file,
+            tx,
         )
         .await;
         let contents = read_to_string(file.path()).unwrap();
@@ -614,7 +652,9 @@ mod tests {
     #[tokio::test(core_threads = 1)]
     /// test that
     async fn banner_needs_update_returns_unknown_with_bad_url() {
-        let result = needs_update(&CONFIGURATION.client, &"", VERSION).await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &"", VERSION, tx).await;
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
@@ -628,7 +668,9 @@ mod tests {
             then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::UpToDate));
@@ -644,7 +686,9 @@ mod tests {
             then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::OutOfDate));
@@ -662,7 +706,9 @@ mod tests {
                 .delay(Duration::from_secs(8));
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
@@ -678,7 +724,9 @@ mod tests {
             then.status(200).body("not json");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
@@ -695,7 +743,9 @@ mod tests {
                 .body("{\"no tag_name\": \"doesn't exist\"}");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));