diff --git a/README.md b/README.md index 178e9346..8510bd94 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Ohayou(おはよう), HTTP load generator, inspired by rakyll/hey with tui anima Usage: oha [OPTIONS] Arguments: - Target URL. + Target URL or file with multiple URLs. Options: -n @@ -122,6 +122,8 @@ Options: Note: If qps is specified, burst will be ignored --rand-regex-url Generate URL by rand_regex crate but dot is disabled for each query e.g. http://127.0.0.1/[a-z][a-z][0-9]. Currently dynamic scheme, host and port with keep-alive do not work well. See https://docs.rs/rand_regex/latest/rand_regex/struct.Regex.html for details of syntax. + --urls-from-file + Read the URLs to query from a file --max-repeat A parameter for the '--rand-regex-url'. The max_repeat parameter gives the maximum extra repeat counts the x*, x+ and x{n,} operators will become. [default: 4] --dump-urls @@ -284,6 +286,22 @@ Optionally you can set `--max-repeat` option to limit max repeat count for each Currently dynamic scheme, host and port with keep-alive are not works well. +## URLs from file feature + +You can use `--urls-from-file` to read the target URLs from a file. Each line of this file needs to contain one valid URL as in the example below. + +``` +http://domain.tld/foo/bar +http://domain.tld/assets/vendors-node_modules_highlight_js_lib_index_js-node_modules_tanstack_react-query_build_modern-3fdf40-591fb51c8a6e.js +http://domain.tld/images/test.png +http://domain.tld/foo/bar?q=test +http://domain.tld/foo +``` + +Such a file can for example be created from an access log to generate a more realistic load distribution over the different pages of a server. + +When this type of URL specification is used, every request goes to a random URL given in the file. + # Contribution Feel free to help us! diff --git a/src/main.rs b/src/main.rs index 6d3bafc4..36ffca83 100644 --- a/src/main.rs +++ b/src/main.rs @@ -34,7 +34,7 @@ static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; #[command(version, about, long_about = None)] #[command(arg_required_else_help(true))] struct Opts { - #[arg(help = "Target URL.")] + #[arg(help = "Target URL or file with multiple URLs.")] url: String, #[arg( help = "Number of requests to run.", @@ -91,6 +91,14 @@ Note: If qps is specified, burst will be ignored", long )] rand_regex_url: bool, + + #[arg( + help = "Read the URLs to query from a file", + default_value = "false", + long + )] + urls_from_file: bool, + #[arg( help = "A parameter for the '--rand-regex-url'. The max_repeat parameter gives the maximum extra repeat counts the x*, x+ and x{n,} operators will become.", default_value = "4", @@ -318,6 +326,8 @@ async fn main() -> anyhow::Result<()> { }) .collect(); UrlGenerator::new_dynamic(Regex::compile(&dot_disabled, opts.max_repeat)?) + } else if opts.urls_from_file { + UrlGenerator::new_multi_static(&opts.url)? } else { UrlGenerator::new_static(Url::parse(&opts.url)?) }; diff --git a/src/url_generator.rs b/src/url_generator.rs index 3e605fa2..1f31bf3e 100644 --- a/src/url_generator.rs +++ b/src/url_generator.rs @@ -1,6 +1,10 @@ +use std::fs::File; +use std::io::{self, BufRead}; +use std::path::Path; use std::{borrow::Cow, string::FromUtf8Error}; use rand::prelude::*; +use rand::seq::SliceRandom; use rand_regex::Regex; use thiserror::Error; use url::{ParseError, Url}; @@ -8,6 +12,7 @@ use url::{ParseError, Url}; #[derive(Clone, Debug)] pub enum UrlGenerator { Static(Url), + MultiStatic(Vec), Dynamic(Regex), } @@ -17,6 +22,10 @@ pub enum UrlGeneratorError { ParseError(ParseError, String), #[error(transparent)] FromUtf8Error(#[from] FromUtf8Error), + #[error("No valid URLs found")] + NoURLsError(), + #[error(transparent)] + IoError(#[from] std::io::Error), } impl UrlGenerator { @@ -24,6 +33,28 @@ impl UrlGenerator { Self::Static(url) } + pub fn new_multi_static(filename: &str) -> Result { + let path = Path::new(filename); + let file = File::open(path)?; + let reader = io::BufReader::new(file); + + let urls: Vec = reader + .lines() + .map_while(Result::ok) + .filter(|line| !line.trim().is_empty()) + .map(|url_str| { + Url::parse(&url_str).map_err(|e| { + UrlGeneratorError::ParseError( + e, + format!("Failed to parse URL '{}': {}", url_str, e), + ) + }) + }) + .collect::, _>>()?; + + Ok(Self::MultiStatic(urls)) + } + pub fn new_dynamic(regex: Regex) -> Self { Self::Dynamic(regex) } @@ -31,6 +62,13 @@ impl UrlGenerator { pub fn generate(&self, rng: &mut R) -> Result, UrlGeneratorError> { match self { Self::Static(url) => Ok(Cow::Borrowed(url)), + Self::MultiStatic(urls) => { + if let Some(random_url) = urls.choose(rng) { + Ok(Cow::Borrowed(random_url)) + } else { + Err(UrlGeneratorError::NoURLsError()) + } + } Self::Dynamic(regex) => { let generated = Distribution::>::sample(regex, rng)?; Ok(Cow::Owned(Url::parse(generated.as_str()).map_err(|e| { @@ -48,7 +86,8 @@ mod tests { use super::*; use rand_regex::Regex as RandRegex; use regex::Regex; - use std::net::Ipv4Addr; + use std::{fs, net::Ipv4Addr}; + use tempfile::NamedTempFile; use url::{Host, Url}; #[test] @@ -59,6 +98,41 @@ mod tests { assert_eq!(url.path(), "/test"); } + #[test] + fn test_url_generator_multistatic() { + let temp_file = NamedTempFile::new().expect("Failed to create temporary file"); + let urls = vec![ + "http://127.0.0.1/a1", + "http://127.0.0.1/b2", + "http://127.0.0.1/c3", + ]; + let file_content = urls.join("\n") + "\n\n"; + fs::write(temp_file.path(), file_content).expect("Failed to write to temporary file"); + + let url_generator = + UrlGenerator::new_multi_static(temp_file.path().to_str().unwrap()).unwrap(); + + for _ in 0..10 { + let url = url_generator.generate(&mut thread_rng()).unwrap(); + assert_eq!(url.host(), Some(Host::Ipv4(Ipv4Addr::new(127, 0, 0, 1)))); + assert!(urls.contains(&url.as_str())); + } + } + + #[test] + fn test_url_generator_multistatic_errors() { + let temp_file = NamedTempFile::new().expect("Failed to create temporary file"); + let file_content = "https://127.0.0.1\n\nno url\n"; + fs::write(temp_file.path(), file_content).expect("Failed to write to temporary file"); + + let url_generator = UrlGenerator::new_multi_static(temp_file.path().to_str().unwrap()); + + assert!( + url_generator.is_err(), + "Parsing should have failed with an error!" + ); + } + #[test] fn test_url_generator_dynamic() { let path_regex = "/[a-z][a-z][0-9]";