Merge pull request #221 from epi052/123-auto-tune-or-bail

added --auto-tune and --auto-bail

.github/pull_request_template.md

@@ -11,7 +11,7 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 
 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic`
 - [ ] All existing tests pass
 
 ## Documentation

.github/workflows/check.yml

@@ -61,4 +61,4 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof
+          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.0.2"
+version = "2.1.0"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -25,7 +25,7 @@ futures = { version = "0.3"}
 tokio = { version = "1.0", features = ["full"] }
 tokio-util = {version = "0.6.3", features = ["codec"]}
 log = "0.4"
-env_logger = "0.8"
+env_logger = "0.8.3"
 reqwest = { version = "0.11", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"

README.md

@@ -8,7 +8,7 @@
 
 <p align="center">
   <a href="https://github.com/epi052/feroxbuster/actions?query=workflow%3A%22CI+Pipeline%22">
-    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/master?logo=github">
+    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/main?logo=github">
   </a>
 
   <a href="https://github.com/epi052/feroxbuster/releases">
@@ -104,6 +104,7 @@ Enumeration.
     - [Cancel a Recursive Scan Interactively (new in `v1.12.0`)](#cancel-a-recursive-scan-interactively-new-in-v1120)
     - [Limit Number of Requests per Second (Rate Limiting) (new in `v2.0.0`)](#limit-number-of-requests-per-second-rate-limiting-new-in-v200)
     - [Silence all Output or Be Kinda Quiet (new in `v2.0.0`)](#silence-all-output-or-be-kinda-quiet-new-in-v200)
+    - [Auto-tune or Auto-bail from Scans (new in `v2.1.0`)](#auto-tune-or-auto-bail-from-scans-new-in-v210)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
@@ -370,6 +371,8 @@ A pre-made configuration file with examples of all available settings can be fou
 # filter_status = [301]
 # threads = 1
 # timeout = 5
+# auto_tune = true
+# auto_bail = true
 # proxy = "http://127.0.0.1:8080"
 # replay_proxy = "http://127.0.0.1:8081"
 # replay_codes = [200, 302]
@@ -425,6 +428,8 @@ USAGE:
 
 FLAGS:
     -f, --add-slash        Append / to each request
+        --auto-bail        Automatically stop scanning when an excessive amount of errors are encountered
+        --auto-tune        Automatically lower scan rate when an excessive amount of errors are encountered
     -D, --dont-filter      Don't auto-filter wildcard responses
     -e, --extract-links    Extract links from response body (html, javascript, etc...); make new requests based on
                            findings (default: false)
@@ -484,7 +489,6 @@ OPTIONS:
     -u, --url <URL>...                            The target URL(s) (required, unless --stdin used)
     -a, --user-agent <USER_AGENT>                 Sets the User-Agent (default: feroxbuster/VERSION)
     -w, --wordlist <FILE>                         Path to the wordlist
-
 ```
 
 ## 📊 Scan's Display Explained
@@ -871,6 +875,29 @@ Scanning: https://localhost.com/homepage
 Scanning: https://localhost.com/api
 ```
 
+### Auto-tune or Auto-bail from scans (new in `v2.1.0`)
+
+Version 2.1.0 introduces the `--auto-tune` and `--auto-bail` flags. You can think of these flags as Policies. Both actions (tuning and bailing) are triggered by the same criteria (below).  Policies are only enforced after at least 50 requests have been made (or # of threads, if that's > 50).
+
+Policy Enforcement Criteria:
+  - number of general errors (timeouts, etc) is higher than half the number of threads (or at least 25 if threads are lower) (per directory scanned)
+  - 90% of responses are `403|Forbidden` (per directory scanned)
+  - 30% of requests are `429|Too Many Requests` (per directory scanned)
+
+> both demo gifs below use --timeout to overload a single-threaded python web server and elicit timeouts
+
+#### --auto-tune
+
+The AutoTune policy enforces a rate limit on individual directory scans when one of the criteria above is met.  The rate limit self-adjusts every (`timeout / 2`) seconds. If the number of errors have increased during that time, the allowed rate of requests is lowered.  On the other hand, if the number of errors hasn't moved, the allowed rate of requests is increased.  If no additional errors are found after a certain number of checks, the rate limit will be removed completely. 
+
+![auto-tune](img/auto-tune-demo.gif)
+
+#### --auto-bail
+
+The AutoBail policy aborts individual directory scans when one of the criteria above is met.  They just stop getting scanned, no muss, no fuss. 
+
+![auto-bail](img/auto-bail-demo.gif)
+
 ## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...
@@ -918,6 +945,8 @@ few of the use-cases in which feroxbuster may be a better fit:
 | cancel a recursive scan interactively (`v1.12.0`)                            | ✔ |   |   |
 | limit number of requests per second (`v2.0.0`)                               | ✔ | ✔ | ✔ |
 | hide progress bars or be silent (or some variation) (`v2.0.0`)               | ✔ | ✔ | ✔ |
+| automatically tune scans based on errors/403s/429s  (`v2.1.0`)               | ✔ |   |   |
+| automatically stop scans based on errors/403s/429s  (`v2.1.0`)               | ✔ |   | ✔ |
 | **huge** number of other options                                             |   |   | ✔ |
 
 Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I

ferox-config.toml.example

@@ -20,6 +20,8 @@
 # rate_limit = 250
 # quiet = true
 # silent = true
+# auto_tune = true
+# auto_bail = true
 # json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"

shell_completions/_feroxbuster

@@ -58,13 +58,15 @@ _feroxbuster() {
 '*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
 '-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
 '--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
-'--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
+'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
 '--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
 '(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
 '-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
+'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
 '-D[Don'\''t auto-filter wildcard responses]' \
 '--dont-filter[Don'\''t auto-filter wildcard responses]' \

shell_completions/_feroxbuster.ps1

@@ -70,6 +70,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
             [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
             [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
+            [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
             [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
             [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
             [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')

shell_completions/feroxbuster.bash

@@ -20,7 +20,7 @@ _feroxbuster() {
 
     case "${cmd}" in
         feroxbuster)
-            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --rate-limit --time-limit  "
+            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --rate-limit --time-limit  "
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0

shell_completions/feroxbuster.fish

@@ -26,6 +26,8 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total
 complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l silent -d 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automatically lower scan rate when an excessive amount of errors are encountered'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered'
 complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
 complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
 complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'

src/banner/container.rs

@@ -1,8 +1,8 @@
 use super::entry::BannerEntry;
-use crate::event_handlers::Handles;
 use crate::{
     config::Configuration,
-    utils::{make_request, status_colorizer},
+    event_handlers::Handles,
+    utils::{logged_request, status_colorizer},
     VERSION,
 };
 use anyhow::{bail, Result};
@@ -125,6 +125,12 @@ pub struct Banner {
     /// represents Configuration.rate_limit
     rate_limit: BannerEntry,
 
+    /// represents Configuration.auto_tune
+    auto_tune: BannerEntry,
+
+    /// represents Configuration.auto_bail
+    auto_bail: BannerEntry,
+
     /// current version of feroxbuster
     pub(super) version: String,
 
@@ -251,6 +257,8 @@ impl Banner {
         );
 
         let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
+        let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string());
+        let auto_bail = BannerEntry::new("🪣", "Auto Bail", &config.auto_bail.to_string());
         let cfg = BannerEntry::new("💉", "Config File", &config.config);
         let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
         let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
@@ -284,6 +292,8 @@ impl Banner {
             filter_status,
             timeout,
             user_agent,
+            auto_bail,
+            auto_tune,
             proxy,
             replay_codes,
             replay_proxy,
@@ -354,15 +364,8 @@ by Ben "epi" Risher {}                 ver: {}"#,
 
         let api_url = Url::parse(url)?;
 
-        let response = make_request(
-            &handles.config.client,
-            &api_url,
-            handles.config.output_level,
-            handles.stats.tx.clone(),
-        )
-        .await?;
-
-        let body = response.text().await?;
+        let result = logged_request(&api_url, handles.clone()).await?;
+        let body = result.text().await?;
 
         let json_response: Value = serde_json::from_str(&body)?;
 
@@ -486,6 +489,13 @@ by Ben "epi" Risher {}                 ver: {}"#,
             writeln!(&mut writer, "{}", self.insecure)?;
         }
 
+        if config.auto_bail {
+            writeln!(&mut writer, "{}", self.auto_bail)?;
+        }
+        if config.auto_tune {
+            writeln!(&mut writer, "{}", self.auto_tune)?;
+        }
+
         if config.redirects {
             writeln!(&mut writer, "{}", self.redirects)?;
         }

src/banner/tests.rs

@@ -1,6 +1,6 @@
 use super::container::UpdateStatus;
 use super::*;
-use crate::{config::Configuration, event_handlers::Handles};
+use crate::{config::Configuration, event_handlers::Handles, scan_manager::FeroxScans};
 use httpmock::Method::GET;
 use httpmock::MockServer;
 use std::{io::stderr, sync::Arc, time::Duration};
@@ -73,8 +73,9 @@ async fn banner_needs_update_returns_up_to_date() {
         when.method(GET).path("/latest");
         then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
     });
+    let scans = Arc::new(FeroxScans::default());
 
-    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
 
     let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
     banner.version = String::from("1.1.0");
@@ -95,7 +96,9 @@ async fn banner_needs_update_returns_out_of_date() {
         then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
     });
 
-    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let scans = Arc::new(FeroxScans::default());
+
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
 
     let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
     banner.version = String::from("1.0.1");

src/config/container.rs

@@ -1,8 +1,9 @@
 use super::utils::{
     depth, report_and_exit, save_state, serialized_type, status_codes, threads, timeout,
-    user_agent, wordlist, OutputLevel,
+    user_agent, wordlist, OutputLevel, RequesterPolicy,
 };
 use crate::config::determine_output_level;
+use crate::config::utils::determine_requester_policy;
 use crate::{
     client, parser, scan_manager::resume_scan, traits::FeroxSerialize, utils::fmt_err,
     DEFAULT_CONFIG_NAME,
@@ -124,6 +125,18 @@ pub struct Configuration {
     #[serde(skip)]
     pub output_level: OutputLevel,
 
+    /// automatically bail at certain error thresholds
+    #[serde(default)]
+    pub auto_bail: bool,
+
+    /// automatically try to lower request rate in order to reduce errors
+    #[serde(default)]
+    pub auto_tune: bool,
+
+    /// more easily differentiate between the three requester policies
+    #[serde(skip)]
+    pub requester_policy: RequesterPolicy,
+
     /// Store log output as NDJSON
     #[serde(default)]
     pub json: bool,
@@ -245,6 +258,7 @@ impl Default for Configuration {
         let replay_codes = status_codes.clone();
         let kind = serialized_type();
         let output_level = OutputLevel::Default;
+        let requester_policy = RequesterPolicy::Default;
 
         Configuration {
             kind,
@@ -254,7 +268,10 @@ impl Default for Configuration {
             replay_codes,
             status_codes,
             replay_client,
+            requester_policy,
             dont_filter: false,
+            auto_bail: false,
+            auto_tune: false,
             silent: false,
             quiet: false,
             output_level,
@@ -313,6 +330,8 @@ impl Configuration {
     /// - **debug_log**: `None`
     /// - **quiet**: `false`
     /// - **silent**: `false`
+    /// - **auto_tune**: `false`
+    /// - **auto_bail**: `false`
     /// - **save_state**: `true`
     /// - **user_agent**: `feroxbuster/VERSION`
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
@@ -561,6 +580,16 @@ impl Configuration {
             config.output_level = OutputLevel::Quiet;
         }
 
+        if args.is_present("auto_tune") {
+            config.auto_tune = true;
+            config.requester_policy = RequesterPolicy::AutoTune;
+        }
+
+        if args.is_present("auto_bail") {
+            config.auto_bail = true;
+            config.requester_policy = RequesterPolicy::AutoBail;
+        }
+
         if args.is_present("dont_filter") {
             config.dont_filter = true;
         }
@@ -721,8 +750,11 @@ impl Configuration {
         update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
         update_if_not_default!(&mut conf.silent, new.silent, false);
         update_if_not_default!(&mut conf.quiet, new.quiet, false);
-        // use updated quiet/silent values to determin output level
+        update_if_not_default!(&mut conf.auto_bail, new.auto_bail, false);
+        update_if_not_default!(&mut conf.auto_tune, new.auto_tune, false);
+        // use updated quiet/silent values to determine output level; same for requester policy
         conf.output_level = determine_output_level(conf.quiet, conf.silent);
+        conf.requester_policy = determine_requester_policy(conf.auto_tune, conf.auto_bail);
         update_if_not_default!(&mut conf.output, new.output, "");
         update_if_not_default!(&mut conf.redirects, new.redirects, false);
         update_if_not_default!(&mut conf.insecure, new.insecure, false);

src/config/mod.rs

@@ -6,4 +6,4 @@ mod utils;
 mod tests;
 
 pub use self::container::Configuration;
-pub use self::utils::{determine_output_level, OutputLevel};
+pub use self::utils::{determine_output_level, OutputLevel, RequesterPolicy};