Support for custom HTTP Headers (#53)

* feat(downloader): support for custom headers

* chore(deps): upgrade and simplify dependencies

* chore(version): revert to 2.1.1

* chore(fmt): reformat

* test(doc): url::Url  -> reqwest::Url

Cargo.toml

@@ -11,20 +11,19 @@ categories = ["concurrency"]
 keywords = ["http", "download", "async", "tokio", "indicatif"]
 
 [dependencies]
-futures = "0.3.21"
-indicatif = "0.17.2"
-reqwest = { version = "0.11.9", features = ["stream", "socks"] }
+futures = "0.3.25"
+indicatif = "0.17.3"
+reqwest = { version = "0.11.13", features = ["stream", "socks"] }
 reqwest-middleware = "0.2.0"
 reqwest-retry = "0.2.1"
 reqwest-tracing = { version = "0.4.0", features = ["opentelemetry_0_17"] }
-task-local-extensions = "0.1.0"
-thiserror = "1.0.30"
+task-local-extensions = "0.1.3"
+thiserror = "1.0.38"
 tracing = "0.1"
 tracing-opentelemetry = "0.18"
 tracing-subscriber = "0.3"
-tokio = { version = "1.17.0", features = ["macros", "rt-multi-thread"] }
-url = "2.2.2"
-http = "0.2.6"
+tokio = { version = "1.24.1", features = ["macros", "rt-multi-thread"] }
+form_urlencoded = "1.1.0"
 
 [dev-dependencies]
 color-eyre = "0.6.1"

examples/with-report.rs

@@ -66,7 +66,6 @@ fn display_summary(summaries: &[Summary]) {
             &status,
             &error,
         ]);
-        ()
     });
     println!("{table}");
 }

examples/with-resume.rs

@@ -6,17 +6,20 @@
 //! cargo run -q --example with-resume
 //! ```
 
-use color_eyre::{eyre::eyre, eyre::Report, Result};
+use color_eyre::{
+    eyre::{eyre, Report},
+    Result,
+};
 use futures::stream::StreamExt;
 use rand::Rng;
-use reqwest::header::{ACCEPT_RANGES, RANGE};
-use std::fs;
-use std::path::PathBuf;
+use reqwest::{
+    header::{ACCEPT_RANGES, RANGE},
+    Url,
+};
+use std::{fs, path::PathBuf};
 use tokio::{fs::File, io::AsyncWriteExt};
 use tracing::debug;
-use tracing_subscriber;
 use trauma::{download::Download, downloader::DownloaderBuilder};
-use url::Url;
 
 #[tokio::main]
 async fn main() -> Result<(), Report> {
@@ -47,7 +50,7 @@ async fn main() -> Result<(), Report> {
     tracing::debug!("Is the file resumable: {:?}", &resumable);
 
     // We must ensure that the download is resumable to prove our point.
-    assert_eq!(resumable, true);
+    assert!(resumable);
 
     // Request a random amount of data to simulate a previously failed download.
     let mut rng = rand::thread_rng();

examples/with-style.rs

@@ -33,13 +33,10 @@ async fn main() -> Result<(), Error> {
         // - https://changaco.oy.lc/unicode-progress-bars/
         // - https://emojistock.com/circle-symbols/
         ProgressBarOpts::new(
-            Some(
-                format!(
-                    "{{bar:40.cyan/blue}} {{percent:>2.magenta}}{} ● {{eta_precise:.blue}}",
-                    style("%").magenta(),
-                )
-                .into(),
-            ),
+            Some(format!(
+                "{{bar:40.cyan/blue}} {{percent:>2.magenta}}{} ● {{eta_precise:.blue}}",
+                style("%").magenta(),
+            )),
             Some("●◕◑◔○".into()),
             true,
             false,

src/download.rs

@@ -1,11 +1,9 @@
 //! Represents a file to be downloaded.
 
 use crate::Error;
-use http::header::ACCEPT_RANGES;
-use http::StatusCode;
+use reqwest::{header::ACCEPT_RANGES, StatusCode, Url};
 use reqwest_middleware::ClientWithMiddleware;
 use std::convert::TryFrom;
-use url::Url;
 
 /// Represents a file to be downloaded.
 #[derive(Debug, Clone)]
@@ -30,7 +28,7 @@ impl Download {
     /// ```no_run
     /// # use color_eyre::{eyre::Report, Result};
     /// use trauma::download::Download;
-    /// use url::Url;
+    /// use reqwest::Url;
     ///
     /// # fn main() -> Result<(), Report> {
     /// Download::try_from("https://example.com/file-0.1.2.zip")?;
@@ -76,7 +74,7 @@ impl TryFrom<&Url> for Download {
             .map(String::from)
             .map(|filename| Download {
                 url: value.clone(),
-                filename: url::form_urlencoded::parse(filename.as_bytes())
+                filename: form_urlencoded::parse(filename.as_bytes())
                     .map(|(key, val)| [key, val].concat())
                     .collect(),
             })
@@ -180,7 +178,7 @@ impl Summary {
 mod test {
     use super::*;
 
-    const DOMAIN: &'static str = "http://domain.com/file.zip";
+    const DOMAIN: &str = "http://domain.com/file.zip";
 
     #[test]
     fn test_try_from_url() {

src/downloader.rs

@@ -1,9 +1,12 @@
 //! Represents the download controller.
 
-use crate::{download::Download, download::Status, download::Summary};
+use crate::download::{Download, Status, Summary};
 use futures::stream::{self, StreamExt};
-use http::{header::RANGE, StatusCode};
 use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
+use reqwest::{
+    header::{HeaderMap, HeaderValue, IntoHeaderName, RANGE},
+    StatusCode,
+};
 use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
 use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware};
 use reqwest_tracing::TracingMiddleware;
@@ -36,6 +39,7 @@ pub struct Downloader {
     style_options: StyleOptions,
     /// Resume the download if necessary and possible.
     resumable: bool,
+    headers: Option<HeaderMap>,
 }
 
 impl Downloader {
@@ -159,6 +163,11 @@ impl Downloader {
         if self.resumable && can_resume {
             req = req.header(RANGE, format!("bytes={}-", size_on_disk));
         }
+
+        if let Some(ref h) = self.headers {
+            req = req.headers(h.to_owned());
+        }
+
         let res = match req.send().await {
             Ok(res) => res,
             Err(e) => {
@@ -306,6 +315,79 @@ impl DownloaderBuilder {
         self
     }
 
+    fn new_header(&self) -> HeaderMap {
+        match self.0.headers {
+            Some(ref h) => h.to_owned(),
+            _ => HeaderMap::new(),
+        }
+    }
+
+    /// Add the http headers.
+    ///
+    /// You need to pass in a `HeaderMap`, not a `HeaderName`.
+    /// `HeaderMap` is a set of http headers.
+    ///
+    /// You can call `.headers()` multiple times and all `HeaderMap` will be merged into a single one.
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// use reqwest::header::{self, HeaderValue, HeaderMap};
+    /// use trauma::downloader::DownloaderBuilder;
+    ///
+    /// let ua = HeaderValue::from_str("curl/7.87").expect("Invalid UA");
+    ///
+    /// let builder = DownloaderBuilder::new()
+    ///     .headers(HeaderMap::from_iter([(header::USER_AGENT, ua)]))
+    ///     .build();
+    /// ```
+    ///
+    /// See also [`header()`].
+    ///
+    /// [`header()`]: DownloaderBuilder::header
+    pub fn headers(mut self, headers: HeaderMap) -> Self {
+        let mut new = self.new_header();
+        new.extend(headers);
+
+        self.0.headers = Some(new);
+        self
+    }
+
+    /// Add the http header
+    ///
+    /// # Example
+    ///
+    /// You can use the `.header()` chain to add multiple headers
+    ///
+    /// ```
+    /// use reqwest::header::{self, HeaderValue};
+    /// use trauma::downloader::DownloaderBuilder;
+    ///
+    /// const FIREFOX_UA: &str =
+    /// "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/109.0";
+    ///
+    /// let ua = HeaderValue::from_str(FIREFOX_UA).expect("Invalid UA");
+    /// let auth = HeaderValue::from_str("Basic aGk6MTIzNDU2Cg==").expect("Invalid auth");
+    ///
+    /// let builder = DownloaderBuilder::new()
+    ///     .header(header::USER_AGENT, ua)
+    ///     .header(header::AUTHORIZATION, auth)
+    ///     .build();
+    /// ```
+    ///
+    /// If you need to pass in a `HeaderMap`, instead of calling `.header()` multiple times.
+    /// See also [`headers()`].
+    ///
+    /// [`headers()`]: DownloaderBuilder::headers
+    pub fn header<K: IntoHeaderName>(mut self, name: K, value: HeaderValue) -> Self {
+        let mut new = self.new_header();
+
+        new.insert(name, value);
+
+        self.0.headers = Some(new);
+        self
+    }
+
     /// Create the [`Downloader`] with the specified options.
     pub fn build(self) -> Downloader {
         Downloader {
@@ -314,6 +396,7 @@ impl DownloaderBuilder {
             concurrent_downloads: self.0.concurrent_downloads,
             style_options: self.0.style_options,
             resumable: self.0.resumable,
+            headers: self.0.headers,
         }
     }
 }
@@ -326,6 +409,7 @@ impl Default for DownloaderBuilder {
             concurrent_downloads: Downloader::DEFAULT_CONCURRENT_DOWNLOADS,
             style_options: StyleOptions::default(),
             resumable: true,
+            headers: None,
         })
     }
 }