cr

iroh-gateway/Cargo.toml

@@ -58,6 +58,7 @@ hex-literal = "0.3.4"
 hex = "0.4.3"
 http-body = "0.4.5"
 mime_classifier = "0.0.1"
+mime = "0.3"
 
 [dev-dependencies]
 axum-macros = "0.2.0" # use #[axum_macros::debug_handler] for better error messages on handlers

iroh-gateway/src/client.rs

@@ -15,6 +15,7 @@ use iroh_metrics::{
 use iroh_resolver::resolver::{
     CidOrDomain, ContentLoader, Metadata, Out, OutMetrics, OutPrettyReader, Resolver, Source,
 };
+use mime::Mime;
 use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWrite};
 use tokio_util::io::ReaderStream;
 use tracing::{info, warn};
@@ -30,7 +31,7 @@ pub struct Client<T: ContentLoader> {
 pub struct PrettyStreamBody<T: ContentLoader>(
     ReaderStream<tokio::io::BufReader<OutPrettyReader<T>>>,
     Option<u64>,
-    Vec<u8>,
+    Option<Mime>,
 );
 
 #[allow(clippy::large_enum_variant)]
@@ -40,7 +41,7 @@ pub enum FileResult<T: ContentLoader> {
 }
 
 impl<T: ContentLoader> PrettyStreamBody<T> {
-    pub fn get_sample(&self) -> Vec<u8> {
+    pub fn get_mime(&self) -> Option<Mime> {
         self.2.clone()
     }
 }
@@ -109,14 +110,11 @@ impl<T: ContentLoader + std::marker::Unpin> Client<T> {
                 .map_err(|e| e.to_string())?;
 
             let mut buf_reader = tokio::io::BufReader::with_capacity(1024 * 1024, reader);
-            let body_sample = buf_reader
-                .fill_buf()
-                .await
-                .map_err(|e| e.to_string())?
-                .to_vec();
+            let body_sample = buf_reader.fill_buf().await.map_err(|e| e.to_string())?;
+            let mime = sniff_content_type(body_sample);
             let stream = ReaderStream::new(buf_reader);
 
-            let body = PrettyStreamBody(stream, metadata.size, body_sample);
+            let body = PrettyStreamBody(stream, metadata.size, Some(mime));
 
             Ok((FileResult::File(body), metadata))
         }
@@ -253,3 +251,18 @@ fn record_ttfb_metrics(start_time: std::time::Instant, source: &Source) {
         );
     }
 }
+
+pub(crate) fn sniff_content_type(body_sample: &[u8]) -> Mime {
+    let classifier = mime_classifier::MimeClassifier::new();
+    let context = mime_classifier::LoadContext::Browsing;
+    let no_sniff_flag = mime_classifier::NoSniffFlag::Off;
+    let apache_bug_flag = mime_classifier::ApacheBugFlag::On;
+    let supplied_type = None;
+    classifier.classify(
+        context,
+        no_sniff_flag,
+        apache_bug_flag,
+        &supplied_type,
+        body_sample,
+    )
+}

iroh-gateway/src/handlers.rs

@@ -536,8 +536,8 @@ async fn serve_fs<T: ContentLoader + std::marker::Unpin>(
                             HeaderValue::from_str("inode/symlink").unwrap(),
                         );
                     } else {
-                        let body_sample = body.get_sample();
-                        add_content_type_headers(&mut headers, &name, body_sample.as_slice());
+                        let content_sniffed_mime = body.get_mime();
+                        add_content_type_headers(&mut headers, &name, content_sniffed_mime);
                     }
                     response(StatusCode::OK, body, headers)
                 }
@@ -567,7 +567,7 @@ async fn serve_fs_dir<T: ContentLoader + std::marker::Unpin>(
             .unwrap_or_default()
     });
     if !force_dir && has_index {
-        if !req.resolved_path.is_dir() {
+        if !req.resolved_path.is_dir_like_path() {
             let redirect_path = format!(
                 "{}/{}",
                 req.resolved_path,
@@ -586,7 +586,7 @@ async fn serve_fs_dir<T: ContentLoader + std::marker::Unpin>(
 
     let mut template_data: Map<String, Json> = Map::new();
     let mut root_path = req.resolved_path.clone();
-    if !root_path.is_dir() {
+    if !root_path.is_dir_like_path() {
         root_path.push("");
     }
     let links = dir_list

iroh-gateway/src/headers.rs

@@ -2,6 +2,7 @@ use crate::{constants::*, response::ResponseFormat};
 use ::time::OffsetDateTime;
 use axum::http::header::*;
 use iroh_resolver::resolver::{CidOrDomain, Metadata, PathType};
+use mime::Mime;
 use std::{fmt::Write, time};
 
 #[tracing::instrument()]
@@ -10,19 +11,25 @@ pub fn add_user_headers(headers: &mut HeaderMap, user_headers: HeaderMap) {
 }
 
 #[tracing::instrument()]
-pub fn add_content_type_headers(headers: &mut HeaderMap, name: &str, body_sample: &[u8]) {
+pub fn add_content_type_headers(
+    headers: &mut HeaderMap,
+    name: &str,
+    content_sniffed_mime: Option<Mime>,
+) {
     let guess = mime_guess::from_path(name);
-    let mut content_type: String;
+    let mut content_type = String::new();
     if let Some(ct) = guess.first() {
         content_type = ct.to_string();
-    } else {
-        content_type = sniff_content_type(body_sample);
+    } else if let Some(ct) = content_sniffed_mime {
+        content_type = ct.to_string();
     }
 
-    if content_type.starts_with("text/") {
+    // for most text types we want to add charset=utf-8
+    if content_type.starts_with("text/") && !content_type.contains("charset") {
         content_type.push_str("; charset=utf-8");
     }
 
+    // for html we want to explicitly have the browser detect encoding
     if content_type.starts_with("text/html") {
         content_type = "text/html".to_string()
     }
@@ -32,24 +39,6 @@ pub fn add_content_type_headers(headers: &mut HeaderMap, name: &str, body_sample
     }
 }
 
-#[tracing::instrument()]
-fn sniff_content_type(body_sample: &[u8]) -> String {
-    let classifier = mime_classifier::MimeClassifier::new();
-    let context = mime_classifier::LoadContext::Browsing;
-    let no_sniff_flag = mime_classifier::NoSniffFlag::Off;
-    let apache_bug_flag = mime_classifier::ApacheBugFlag::On;
-    let supplied_type = None;
-    let computed_type = classifier.classify(
-        context,
-        no_sniff_flag,
-        apache_bug_flag,
-        &supplied_type,
-        body_sample,
-    );
-
-    computed_type.to_string()
-}
-
 #[tracing::instrument()]
 pub fn add_content_disposition_headers(
     headers: &mut HeaderMap,
@@ -224,7 +213,8 @@ mod tests {
         let mut headers = HeaderMap::new();
         let name = "test.txt";
         let body = "test body";
-        add_content_type_headers(&mut headers, name, body.as_bytes());
+        let content_sniffed_mime = Some(crate::client::sniff_content_type(body.as_bytes()));
+        add_content_type_headers(&mut headers, name, content_sniffed_mime.clone());
         assert_eq!(headers.len(), 1);
         assert_eq!(
             headers.get(&CONTENT_TYPE).unwrap(),
@@ -233,7 +223,7 @@ mod tests {
 
         let mut headers = HeaderMap::new();
         let name = "test.RAND_EXT";
-        add_content_type_headers(&mut headers, name, body.as_bytes());
+        add_content_type_headers(&mut headers, name, content_sniffed_mime);
         assert_eq!(headers.len(), 1);
         assert_eq!(
             headers.get(&CONTENT_TYPE).unwrap(),

iroh-resolver/src/resolver.rs

@@ -62,7 +62,8 @@ impl Path {
         &self.tail
     }
 
-    pub fn is_dir(&self) -> bool {
+    // used only for string path manipulation
+    pub fn is_dir_like_path(&self) -> bool {
         !self.tail.is_empty() && self.tail.last().unwrap().is_empty()
     }
 
@@ -78,7 +79,7 @@ impl Path {
             }
             s.push_str(&format!("/{}", part)[..]);
         }
-        if self.is_dir() {
+        if self.is_dir_like_path() {
             s.push('/');
         }
         s
@@ -111,7 +112,7 @@ impl Display for Path {
             write!(f, "/{}", part)?;
         }
 
-        if self.is_dir() {
+        if self.is_dir_like_path() {
             write!(f, "/")?;
         }
 
@@ -1197,8 +1198,8 @@ mod tests {
 
         assert!(non_dir_path.to_string() == non_dir_test);
         assert!(dir_path.to_string() == dir_test);
-        assert!(dir_path.is_dir());
-        assert!(!non_dir_path.is_dir());
+        assert!(dir_path.is_dir_like_path());
+        assert!(!non_dir_path.is_dir_like_path());
     }
 
     fn make_ipld() -> Ipld {