Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

load wordlist from url; change some defaults/fix some bugs #834

Merged
merged 11 commits into from
Mar 18, 2023
Merged

load wordlist from url; change some defaults/fix some bugs #834

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
0b0e08a
updated extract-links and collect-backups default to true
epi052 Mar 17, 2023
92cc2ab
fixed test
epi052 Mar 17, 2023
1b8fdce
hid old false defaults; added dont-* flags
epi052 Mar 17, 2023
d6b10c6
reverted collect-backups change
epi052 Mar 18, 2023
8fbf9d0
-w accepts http/https urls
epi052 Mar 18, 2023
bd4566f
updated parser text
epi052 Mar 18, 2023
c493d00
fmt clippy etc
epi052 Mar 18, 2023
d98c6a7
bumped deps
epi052 Mar 18, 2023
bd54ad0
Merge branch 'main' into 827-load-wordlist-from-url
epi052 Mar 18, 2023
d561e59
added test
epi052 Mar 18, 2023
46d0050
removed cruft
epi052 Mar 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 105 additions & 85 deletions Cargo.lock
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "feroxbuster"
version = "2.9.1"
version = "2.9.2"
authors = ["Ben 'epi' Risher (@epi052)"]
license = "MIT"
edition = "2021"
Expand Down Expand Up @@ -62,7 +62,7 @@ self_update = {version = "0.36.0", features = ["archive-tar", "compression-flate
tempfile = "3.3.0"
httpmock = "0.6.6"
assert_cmd = "2.0.4"
predicates = "2.1.1"
predicates = "3.0.1"

[profile.release]
lto = true
Expand Down
15 changes: 8 additions & 7 deletions shell_completions/_feroxbuster
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ _feroxbuster() {
'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
'*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
'-a+[Sets the User-Agent (default: feroxbuster/2.9.1)]:USER_AGENT: ' \
'--user-agent=[Sets the User-Agent (default: feroxbuster/2.9.1)]:USER_AGENT: ' \
'-a+[Sets the User-Agent (default: feroxbuster/2.9.2)]:USER_AGENT: ' \
'--user-agent=[Sets the User-Agent (default: feroxbuster/2.9.2)]:USER_AGENT: ' \
'*-x+[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
'*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
'*-m+[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \
Expand Down Expand Up @@ -62,8 +62,8 @@ _feroxbuster() {
'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]:PARALLEL_SCANS: ' \
'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]:RATE_LIMIT: ' \
'--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]:TIME_SPEC: ' \
'-w+[Path to the wordlist]:FILE:_files' \
'--wordlist=[Path to the wordlist]:FILE:_files' \
'-w+[Path or URL of the wordlist]:FILE:_files' \
'--wordlist=[Path or URL of the wordlist]:FILE:_files' \
'*-I+[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
'*--dont-collect=[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
'-o+[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
Expand All @@ -72,7 +72,7 @@ _feroxbuster() {
'(-u --url)--stdin[Read url(s) from STDIN]' \
'(-p --proxy -k --insecure --burp-replay)--burp[Set --proxy to http://127.0.0.1:8080 and set --insecure to true]' \
'(-P --replay-proxy -k --insecure)--burp-replay[Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true]' \
'(--rate-limit --auto-bail)--smart[Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true]' \
'(--rate-limit --auto-bail)--smart[Set --auto-tune, --collect-words, and --collect-backups to true]' \
'(--rate-limit --auto-bail)--thorough[Use the same settings as --smart and set --collect-extensions to true]' \
'-A[Use a random User-Agent]' \
'--random-agent[Use a random User-Agent]' \
Expand All @@ -85,8 +85,9 @@ _feroxbuster() {
'-n[Do not scan recursively]' \
'--no-recursion[Do not scan recursively]' \
'(-n --no-recursion)--force-recursion[Force recursion attempts on all '\''found'\'' endpoints (still respects recursion depth)]' \
'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings]' \
'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings]' \
'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)]' \
'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)]' \
'--dont-extract-links[Don'\''t extract links from response body (html, javascript, etc...)]' \
'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
'-D[Don'\''t auto-filter wildcard responses]' \
Expand Down
15 changes: 8 additions & 7 deletions shell_completions/_feroxbuster.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
[CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
[CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
[CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
[CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.1)')
[CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.1)')
[CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.2)')
[CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.2)')
[CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
[CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
[CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
Expand Down Expand Up @@ -68,8 +68,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
[CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
[CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
[CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
[CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path to the wordlist')
[CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path to the wordlist')
[CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
[CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
[CompletionResult]::new('-I', 'I', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
[CompletionResult]::new('--dont-collect', 'dont-collect', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
[CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
Expand All @@ -78,7 +78,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
[CompletionResult]::new('--stdin', 'stdin', [CompletionResultType]::ParameterName, 'Read url(s) from STDIN')
[CompletionResult]::new('--burp', 'burp', [CompletionResultType]::ParameterName, 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true')
[CompletionResult]::new('--burp-replay', 'burp-replay', [CompletionResultType]::ParameterName, 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true')
[CompletionResult]::new('--smart', 'smart', [CompletionResultType]::ParameterName, 'Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true')
[CompletionResult]::new('--smart', 'smart', [CompletionResultType]::ParameterName, 'Set --auto-tune, --collect-words, and --collect-backups to true')
[CompletionResult]::new('--thorough', 'thorough', [CompletionResultType]::ParameterName, 'Use the same settings as --smart and set --collect-extensions to true')
[CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
[CompletionResult]::new('--random-agent', 'random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
Expand All @@ -91,8 +91,9 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
[CompletionResult]::new('-n', 'n', [CompletionResultType]::ParameterName, 'Do not scan recursively')
[CompletionResult]::new('--no-recursion', 'no-recursion', [CompletionResultType]::ParameterName, 'Do not scan recursively')
[CompletionResult]::new('--force-recursion', 'force-recursion', [CompletionResultType]::ParameterName, 'Force recursion attempts on all ''found'' endpoints (still respects recursion depth)')
[CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings')
[CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings')
[CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)')
[CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)')
[CompletionResult]::new('--dont-extract-links', 'dont-extract-links', [CompletionResultType]::ParameterName, 'Don''t extract links from response body (html, javascript, etc...)')
[CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
[CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
[CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
Expand Down
2 changes: 1 addition & 1 deletion shell_completions/feroxbuster.bash
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ _feroxbuster() {

case "${cmd}" in
feroxbuster)
opts="-u -p -P -R -a -A -x -m -H -b -Q -f -S -X -W -N -C -s -T -r -k -t -n -d -e -L -w -D -E -B -g -I -v -q -o -U -h -V --url --stdin --resume-from --burp --burp-replay --smart --thorough --proxy --replay-proxy --replay-codes --user-agent --random-agent --extensions --methods --data --headers --cookies --query --add-slash --dont-scan --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --status-codes --timeout --redirects --insecure --threads --no-recursion --depth --force-recursion --extract-links --scan-limit --parallel --rate-limit --time-limit --wordlist --auto-tune --auto-bail --dont-filter --collect-extensions --collect-backups --collect-words --dont-collect --verbosity --silent --quiet --json --output --debug-log --no-state --update --help --version"
opts="-u -p -P -R -a -A -x -m -H -b -Q -f -S -X -W -N -C -s -T -r -k -t -n -d -e -L -w -D -E -B -g -I -v -q -o -U -h -V --url --stdin --resume-from --burp --burp-replay --smart --thorough --proxy --replay-proxy --replay-codes --user-agent --random-agent --extensions --methods --data --headers --cookies --query --add-slash --dont-scan --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --status-codes --timeout --redirects --insecure --threads --no-recursion --depth --force-recursion --extract-links --dont-extract-links --scan-limit --parallel --rate-limit --time-limit --wordlist --auto-tune --auto-bail --dont-filter --collect-extensions --collect-backups --collect-words --dont-collect --verbosity --silent --quiet --json --output --debug-log --no-state --update --help --version"
if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0
Expand Down
15 changes: 8 additions & 7 deletions shell_completions/feroxbuster.elv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
cand -a 'Sets the User-Agent (default: feroxbuster/2.9.1)'
cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.9.1)'
cand -a 'Sets the User-Agent (default: feroxbuster/2.9.2)'
cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.9.2)'
cand -x 'File extension(s) to search for (ex: -x php -x pdf js)'
cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)'
cand -m 'Which HTTP request method(s) should be sent (default: GET)'
Expand Down Expand Up @@ -65,8 +65,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
cand --parallel 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
cand --rate-limit 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
cand --time-limit 'Limit total run time of all scans (ex: --time-limit 10m)'
cand -w 'Path to the wordlist'
cand --wordlist 'Path to the wordlist'
cand -w 'Path or URL of the wordlist'
cand --wordlist 'Path or URL of the wordlist'
cand -I 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
cand --dont-collect 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
cand -o 'Output file to write results to (use w/ --json for JSON entries)'
Expand All @@ -75,7 +75,7 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
cand --stdin 'Read url(s) from STDIN'
cand --burp 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true'
cand --burp-replay 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true'
cand --smart 'Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true'
cand --smart 'Set --auto-tune, --collect-words, and --collect-backups to true'
cand --thorough 'Use the same settings as --smart and set --collect-extensions to true'
cand -A 'Use a random User-Agent'
cand --random-agent 'Use a random User-Agent'
Expand All @@ -88,8 +88,9 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
cand -n 'Do not scan recursively'
cand --no-recursion 'Do not scan recursively'
cand --force-recursion 'Force recursion attempts on all ''found'' endpoints (still respects recursion depth)'
cand -e 'Extract links from response body (html, javascript, etc...); make new requests based on findings'
cand --extract-links 'Extract links from response body (html, javascript, etc...); make new requests based on findings'
cand -e 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
cand --extract-links 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
cand --dont-extract-links 'Don''t extract links from response body (html, javascript, etc...)'
cand --auto-tune 'Automatically lower scan rate when an excessive amount of errors are encountered'
cand --auto-bail 'Automatically stop scanning when an excessive amount of errors are encountered'
cand -D 'Don''t auto-filter wildcard responses'
Expand Down
21 changes: 10 additions & 11 deletions src/config/container.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use super::utils::{
depth, ignored_extensions, methods, report_and_exit, save_state, serialized_type, status_codes,
threads, timeout, user_agent, wordlist, OutputLevel, RequesterPolicy,
depth, extract_links, ignored_extensions, methods, report_and_exit, save_state,
serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
RequesterPolicy,
};
use crate::config::determine_output_level;
use crate::config::utils::determine_requester_policy;
Expand Down Expand Up @@ -214,7 +215,7 @@ pub struct Configuration {
pub no_recursion: bool,

/// Extract links from html/javscript
#[serde(default)]
#[serde(default = "extract_links")]
pub extract_links: bool,

/// Append / to each request
Expand Down Expand Up @@ -328,6 +329,7 @@ impl Default for Configuration {
let kind = serialized_type();
let output_level = OutputLevel::Default;
let requester_policy = RequesterPolicy::Default;
let extract_links = extract_links();

Configuration {
kind,
Expand All @@ -336,6 +338,7 @@ impl Default for Configuration {
user_agent,
replay_codes,
status_codes,
extract_links,
replay_client,
requester_policy,
dont_filter: false,
Expand All @@ -355,7 +358,6 @@ impl Default for Configuration {
insecure: false,
redirects: false,
no_recursion: false,
extract_links: false,
random_agent: false,
collect_extensions: false,
collect_backups: false,
Expand Down Expand Up @@ -398,7 +400,7 @@ impl Configuration {
///
/// - **timeout**: `5` seconds
/// - **redirects**: `false`
/// - **extract-links**: `false`
/// - **extract_links**: `true`
/// - **wordlist**: [`DEFAULT_WORDLIST`](constant.DEFAULT_WORDLIST.html)
/// - **config**: `None`
/// - **threads**: `50`
Expand Down Expand Up @@ -807,11 +809,8 @@ impl Configuration {
config.add_slash = true;
}

if came_from_cli!(args, "extract_links")
|| came_from_cli!(args, "smart")
|| came_from_cli!(args, "thorough")
{
config.extract_links = true;
if came_from_cli!(args, "dont_extract_links") {
config.extract_links = false;
}

if came_from_cli!(args, "json") {
Expand Down Expand Up @@ -997,7 +996,7 @@ impl Configuration {
update_if_not_default!(&mut conf.redirects, new.redirects, false);
update_if_not_default!(&mut conf.insecure, new.insecure, false);
update_if_not_default!(&mut conf.force_recursion, new.force_recursion, false);
update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
update_if_not_default!(&mut conf.extract_links, new.extract_links, extract_links());
update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
update_if_not_default!(&mut conf.methods, new.methods, methods());
update_if_not_default!(&mut conf.data, new.data, Vec::<u8>::new());
Expand Down
Loading