diff --git a/Cargo.lock b/Cargo.lock index 76c671712..de55dc50b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -677,7 +677,6 @@ dependencies = [ "serde", "serde_json", "slug", - "staticfile", "structopt", "strum", "systemstat", @@ -799,17 +798,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" -[[package]] -name = "filetime" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714653f3e34871534de23771ac7b26e999651a0a228f47beb324dfdf1dd4b10f" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", -] - [[package]] name = "filetime" version = "0.2.10" @@ -1781,16 +1769,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41f5c9112cb662acd3b204077e0de5bc66305fa8df65c8019d5adb10e9ab6e58" -[[package]] -name = "mount" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32245731923cd096899502fc4c4317cfd09f121e80e73f7f576cf3777a824256" -dependencies = [ - "iron", - "sequence_trie", -] - [[package]] name = "native-tls" version = "0.2.4" @@ -1861,7 +1839,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd" dependencies = [ "bitflags", - "filetime 0.2.10", + "filetime", "fsevent", "fsevent-sys", "inotify", @@ -3047,12 +3025,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -[[package]] -name = "sequence_trie" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c915714ca833b1d4d6b8f6a9d72a3ff632fe45b40a8d184ef79c81bec6327eed" - [[package]] name = "serde" version = "1.0.110" @@ -3244,19 +3216,6 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e4b8c631c998468961a9ea159f064c5c8499b95b5e4a34b77849d45949d540" -[[package]] -name = "staticfile" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31493480e073d52522a94cdf56269dd8eb05f99549effd1826b0271690608878" -dependencies = [ - "filetime 0.1.15", - "iron", - "mount", - "time 0.1.43", - "url 1.7.2", -] - [[package]] name = "stdweb" version = "0.4.20" @@ -3453,7 +3412,7 @@ version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" dependencies = [ - "filetime 0.2.10", + "filetime", "libc", "redox_syscall", "xattr", diff --git a/Cargo.toml b/Cargo.toml index 7c87c047a..7758eee4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,7 +66,6 @@ serde_json = "1.0" # iron dependencies iron = "0.5" router = "0.5" -staticfile = { version = "0.4", features = ["cache"] } tempfile = "3.1.0" # Templating diff --git a/build.rs b/build.rs index d1dc77b7e..d398ca7c8 100644 --- a/build.rs +++ b/build.rs @@ -1,11 +1,5 @@ use git2::Repository; -use std::{ - env, - error::Error, - fs::{self, File}, - io::Write, - path::Path, -}; +use std::{env, error::Error, fs::File, io::Write, path::Path}; fn main() { // Don't rerun anytime a single change is made @@ -15,8 +9,6 @@ fn main() { println!("cargo:rerun-if-changed=templates/style/_vars.scss"); println!("cargo:rerun-if-changed=templates/style/_utils.scss"); println!("cargo:rerun-if-changed=templates/style/_navbar.scss"); - println!("cargo:rerun-if-changed=templates/menu.js"); - println!("cargo:rerun-if-changed=templates/index.js"); println!("cargo:rerun-if-changed=vendor/"); // TODO: are these right? println!("cargo:rerun-if-changed=.git/HEAD"); @@ -26,7 +18,6 @@ fn main() { if let Err(sass_err) = compile_sass() { panic!("Error compiling sass: {}", sass_err); } - copy_js(); } fn write_git_version() { @@ -95,12 +86,3 @@ fn compile_sass() -> Result<(), Box> { Ok(()) } - -fn copy_js() { - ["menu.js", "index.js"].iter().for_each(|path| { - let source_path = - Path::new(&env::var("CARGO_MANIFEST_DIR").unwrap()).join(format!("templates/{}", path)); - let dest_path = Path::new(&env::var("OUT_DIR").unwrap()).join(path); - fs::copy(&source_path, &dest_path).expect("Copy JavaScript file to target"); - }); -} diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 20f8aefff..96abcbd3b 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -51,8 +51,6 @@ RUN touch build.rs COPY src src/ RUN find src -name "*.rs" -exec touch {} \; COPY templates/style templates/style -COPY templates/index.js templates/ -COPY templates/menu.js templates/ COPY vendor vendor/ RUN cargo build --release @@ -76,7 +74,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ RUN mkdir -p /opt/docsrs/prefix COPY --from=build /build/target/release/cratesfyi /usr/local/bin -COPY static /opt/docsrs/prefix/public_html +COPY static /opt/docsrs/static COPY templates /opt/docsrs/templates COPY dockerfiles/entrypoint.sh /opt/docsrs/ COPY vendor /opt/docsrs/vendor diff --git a/src/web/error.rs b/src/web/error.rs index b9a8c4479..58a3c3cbd 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -144,7 +144,7 @@ mod tests { .next() .unwrap() .text_contents(), - "The requested resource does not exist", + "The requested crate does not exist", ); Ok(()) @@ -170,7 +170,7 @@ mod tests { .next() .unwrap() .text_contents(), - "The requested resource does not exist", + "The requested crate does not exist", ); Ok(()) @@ -190,7 +190,7 @@ mod tests { .next() .unwrap() .text_contents(), - "The requested resource does not exist", + "The requested crate does not exist", ); Ok(()) @@ -209,7 +209,7 @@ mod tests { .next() .unwrap() .text_contents(), - "The requested resource does not exist", + "The requested crate does not exist", ); Ok(()) @@ -232,7 +232,7 @@ mod tests { .next() .unwrap() .text_contents(), - "The requested resource does not exist", + "The requested crate does not exist", ); Ok(()) diff --git a/src/web/file.rs b/src/web/file.rs index 84d2cc4c6..de9b0ca4f 100644 --- a/src/web/file.rs +++ b/src/web/file.rs @@ -1,7 +1,7 @@ //! Database based file handler use crate::storage::{Blob, Storage}; -use crate::{error::Result, Config}; +use crate::{error::Result, Config, Metrics}; use iron::{status, Handler, IronResult, Request, Response}; #[derive(Debug)] @@ -60,6 +60,15 @@ impl Handler for DatabaseFileHandler { let storage = extension!(req, Storage); let config = extension!(req, Config); if let Ok(file) = File::from_path(&storage, &path, &config) { + let metrics = extension!(req, Metrics); + + // Because all requests that don't hit another handler go through here, we will get all + // requests successful or not recorded by the RequestRecorder, so we inject an extra + // "database success" route to keep track of how often we succeed vs 404 + metrics + .routes_visited + .with_label_values(&["database success"]) + .inc(); Ok(file.serve()) } else { Err(super::error::Nope::CrateNotFound.into()) diff --git a/src/web/metrics.rs b/src/web/metrics.rs index 3c9ec0166..78180333e 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -119,7 +119,7 @@ mod tests { ("/crate/rcc/0.0.0", "/crate/:name/:version"), ("/-/static/index.js", "static resource"), ("/-/static/menu.js", "static resource"), - ("/opensearch.xml", "static resource"), + ("/-/static/opensearch.xml", "static resource"), ("/releases", "/releases"), ("/releases/feed", "static resource"), ("/releases/queue", "/releases/queue"), @@ -129,10 +129,12 @@ mod tests { "/releases/recent-failures/:page", ), ("/releases/recent/1", "/releases/recent/:page"), - ("/robots.txt", "static resource"), + ("/-/static/robots.txt", "static resource"), ("/sitemap.xml", "static resource"), ("/-/static/style.css", "static resource"), ("/-/static/vendored.css", "static resource"), + ("/rustdoc/rcc/0.0.0/rcc/index.html", "database"), + ("/rustdoc/gcc/0.0.0/gcc/index.html", "database"), ]; wrapper(|env| { @@ -179,6 +181,15 @@ mod tests { ); } + // extra metrics for the "database success" hack + assert_eq!( + metrics + .routes_visited + .with_label_values(&["database success"]) + .get(), + 2 + ); + Ok(()) }) } diff --git a/src/web/mod.rs b/src/web/mod.rs index 64136a08f..d26edd6c2 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -96,23 +96,22 @@ use extensions::InjectExtensions; use failure::Error; use iron::{ self, - headers::{CacheControl, CacheDirective, ContentType, Expires, HttpDate}, + headers::{Expires, HttpDate}, modifiers::Redirect, status, status::Status, Chain, Handler, Iron, IronError, IronResult, Listening, Request, Response, Url, }; +use metrics::RequestRecorder; use page::TemplateData; use postgres::Client; use router::NoRoute; use semver::{Version, VersionReq}; use serde::Serialize; -use staticfile::Static; -use std::{borrow::Cow, env, fmt, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration}; +use std::{borrow::Cow, fmt, net::SocketAddr, sync::Arc}; /// Duration of static files for staticfile and DatabaseFileHandler (in seconds) const STATIC_FILE_CACHE_DURATION: u64 = 60 * 60 * 24 * 30 * 12; // 12 months -const OPENSEARCH_XML: &[u8] = include_bytes!("opensearch.xml"); const DEFAULT_BIND: &str = "0.0.0.0:3000"; @@ -120,7 +119,6 @@ struct CratesfyiHandler { shared_resource_handler: Box, router_handler: Box, database_file_handler: Box, - static_handler: Box, inject_extensions: InjectExtensions, } @@ -144,22 +142,14 @@ impl CratesfyiHandler { let shared_resources = Self::chain(inject_extensions.clone(), rustdoc::SharedResourceHandler); let router_chain = Self::chain(inject_extensions.clone(), routes.iron_router()); - let prefix = PathBuf::from( - env::var("CRATESFYI_PREFIX") - .expect("the CRATESFYI_PREFIX environment variable is not set"), - ) - .join("public_html"); - let static_handler = - Static::new(prefix).cache(Duration::from_secs(STATIC_FILE_CACHE_DURATION)); Ok(CratesfyiHandler { shared_resource_handler: Box::new(shared_resources), router_handler: Box::new(router_chain), database_file_handler: Box::new(routes::BlockBlacklistedPrefixes::new( blacklisted_prefixes, - Box::new(file::DatabaseFileHandler), + Box::new(RequestRecorder::new(file::DatabaseFileHandler, "database")), )), - static_handler: Box::new(static_handler), inject_extensions, }) } @@ -185,7 +175,6 @@ impl Handler for CratesfyiHandler { .handle(req) .or_else(|e| if_404(e, || self.router_handler.handle(req))) .or_else(|e| if_404(e, || self.database_file_handler.handle(req))) - .or_else(|e| if_404(e, || self.static_handler.handle(req))) .or_else(|e| { let err = if let Some(err) = e.error.downcast::() { *err @@ -518,21 +507,6 @@ fn redirect_base(req: &Request) -> String { } } -fn opensearch_xml_handler(_: &mut Request) -> IronResult { - let mut response = Response::with((status::Ok, OPENSEARCH_XML)); - let cache = vec![ - CacheDirective::Public, - CacheDirective::MaxAge(STATIC_FILE_CACHE_DURATION as u32), - ]; - response.headers.set(ContentType( - "application/opensearchdescription+xml".parse().unwrap(), - )); - - response.headers.set(CacheControl(cache)); - - Ok(response) -} - /// MetaData used in header #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub(crate) struct MetaData { diff --git a/src/web/routes.rs b/src/web/routes.rs index 78c6b4ad9..935049fe9 100644 --- a/src/web/routes.rs +++ b/src/web/routes.rs @@ -10,9 +10,21 @@ pub(super) const DOC_RUST_LANG_ORG_REDIRECTS: &[&str] = pub(super) fn build_routes() -> Routes { let mut routes = Routes::new(); - routes.static_resource("/robots.txt", super::sitemap::robots_txt_handler); + // Well known resources, robots.txt and favicon.ico support redirection, the sitemap.xml + // must live at the site root: + // https://developers.google.com/search/reference/robots_txt#handling-http-result-codes + // https://support.google.com/webmasters/answer/183668?hl=en + routes.static_resource("/robots.txt", PermanentRedirect("/-/static/robots.txt")); + routes.static_resource("/favicon.ico", PermanentRedirect("/-/static/favicon.ico")); routes.static_resource("/sitemap.xml", super::sitemap::sitemap_handler); - routes.static_resource("/opensearch.xml", super::opensearch_xml_handler); + + // This should not need to be served from the root as we reference the inner path in links, + // but clients might have cached the url and need to update it. + routes.static_resource( + "/opensearch.xml", + PermanentRedirect("/-/static/opensearch.xml"), + ); + routes.static_resource("/-/static/:file", super::statics::static_handler); routes.internal_page("/", super::releases::home_page); @@ -257,7 +269,21 @@ impl Handler for SimpleRedirect { (self.url_mangler)(&mut url); Ok(iron::Response::with(( iron::status::Found, - iron::modifiers::Redirect(iron::Url::parse(&url.to_string()).unwrap()), + iron::modifiers::Redirect(iron::Url::from_generic_url(url).unwrap()), + ))) + } +} + +#[derive(Copy, Clone)] +struct PermanentRedirect(&'static str); + +impl Handler for PermanentRedirect { + fn handle(&self, req: &mut iron::Request) -> iron::IronResult { + let mut url: iron::url::Url = req.url.clone().into(); + url.set_path(self.0); + Ok(iron::Response::with(( + iron::status::MovedPermanently, + iron::modifiers::Redirect(iron::Url::from_generic_url(url).unwrap()), ))) } } @@ -299,3 +325,29 @@ fn calculate_id(pattern: &str) -> String { .map(|c| if c.is_alphanumeric() { c } else { '_' }) .collect() } + +#[cfg(test)] +mod tests { + use crate::test::*; + + #[test] + fn test_root_redirects() { + wrapper(|env| { + // These are "well-known" resources that will be requested from the root, but support + // redirection + assert_redirect("/favicon.ico", "/-/static/favicon.ico", env.frontend())?; + assert_redirect("/robots.txt", "/-/static/robots.txt", env.frontend())?; + + // This has previously been served with a url pointing to the root, it may be + // plausible to remove the redirects in the future, but for now we need to keep serving + // it. + assert_redirect( + "/opensearch.xml", + "/-/static/opensearch.xml", + env.frontend(), + )?; + + Ok(()) + }); + } +} diff --git a/src/web/sitemap.rs b/src/web/sitemap.rs index 41152c297..3e56c1b78 100644 --- a/src/web/sitemap.rs +++ b/src/web/sitemap.rs @@ -3,7 +3,7 @@ use chrono::{DateTime, NaiveDateTime, Utc}; use iron::{ headers::ContentType, mime::{Mime, SubLevel, TopLevel}, - status, IronResult, Request, Response, + IronResult, Request, Response, }; use serde::Serialize; use serde_json::Value; @@ -48,13 +48,6 @@ pub fn sitemap_handler(req: &mut Request) -> IronResult { SitemapXml { releases }.into_response(req) } -pub fn robots_txt_handler(_: &mut Request) -> IronResult { - let mut resp = Response::with((status::Ok, "Sitemap: https://docs.rs/sitemap.xml")); - resp.headers.set(ContentType::plaintext()); - - Ok(resp) -} - #[derive(Debug, Clone, PartialEq, Eq, Serialize)] struct AboutBuilds { /// The current version of rustc that docs.rs is using to build crates diff --git a/src/web/statics.rs b/src/web/statics.rs index 4758eb658..f737ee3c9 100644 --- a/src/web/statics.rs +++ b/src/web/statics.rs @@ -12,28 +12,17 @@ use std::{ffi::OsStr, fs, path::Path}; const VENDORED_CSS: &str = include_str!(concat!(env!("OUT_DIR"), "/vendored.css")); const STYLE_CSS: &str = include_str!(concat!(env!("OUT_DIR"), "/style.css")); -const MENU_JS: &str = include_str!(concat!(env!("OUT_DIR"), "/menu.js")); -const INDEX_JS: &str = include_str!(concat!(env!("OUT_DIR"), "/index.js")); -const STATIC_SEARCH_PATHS: &[&str] = &["vendor/pure-css/css"]; +const STATIC_SEARCH_PATHS: &[&str] = &["vendor/pure-css/css", "static"]; pub(crate) fn static_handler(req: &mut Request) -> IronResult { let router = extension!(req, Router); let file = cexpect!(req, router.find("file")); - match file { - "vendored.css" => serve_resource(VENDORED_CSS, ContentType("text/css".parse().unwrap())), - "style.css" => serve_resource(STYLE_CSS, ContentType("text/css".parse().unwrap())), - "index.js" => serve_resource( - INDEX_JS, - ContentType("application/javascript".parse().unwrap()), - ), - "menu.js" => serve_resource( - MENU_JS, - ContentType("application/javascript".parse().unwrap()), - ), - - file => serve_file(req, file), - } + Ok(match file { + "vendored.css" => serve_resource(VENDORED_CSS, ContentType("text/css".parse().unwrap()))?, + "style.css" => serve_resource(STYLE_CSS, ContentType("text/css".parse().unwrap()))?, + file => serve_file(req, file)?, + }) } fn serve_file(req: &Request, file: &str) -> IronResult { @@ -53,7 +42,7 @@ fn serve_file(req: &Request, file: &str) -> IronResult { // If we can detect the file's mime type, set it // MimeGuess misses a lot of the file types we need, so there's a small wrapper // around it - let content_type = path + let mut content_type = path .extension() .and_then(OsStr::to_str) .and_then(|ext| match ext { @@ -68,6 +57,12 @@ fn serve_file(req: &Request, file: &str) -> IronResult { .map(|mime| ContentType(mime.as_ref().parse().unwrap())), }); + if file == "opensearch.xml" { + content_type = Some(ContentType( + "application/opensearchdescription+xml".parse().unwrap(), + )); + } + serve_resource(contents, content_type) } @@ -104,8 +99,8 @@ where pub(super) fn ico_handler(req: &mut Request) -> IronResult { if let Some(&"favicon.ico") = req.url.path().last() { - // if we're looking for exactly "favicon.ico", we need to defer to the handler that loads - // from `public_html`, so return a 404 here to make the main handler carry on + // if we're looking for exactly "favicon.ico", we need to defer to the handler that + // actually serves it, so return a 404 here to make the main handler carry on Err(Nope::ResourceNotFound.into()) } else { // if we're looking for something like "favicon-20190317-1.35.0-nightly-c82834e2b.ico", @@ -121,7 +116,7 @@ pub(super) fn ico_handler(req: &mut Request) -> IronResult { #[cfg(test)] mod tests { - use super::{INDEX_JS, MENU_JS, STATIC_SEARCH_PATHS, STYLE_CSS, VENDORED_CSS}; + use super::{STATIC_SEARCH_PATHS, STYLE_CSS, VENDORED_CSS}; use crate::test::wrapper; use std::fs; @@ -172,8 +167,8 @@ mod tests { resp.headers().get("Content-Type"), Some(&"application/javascript".parse().unwrap()), ); - assert_eq!(resp.content_length().unwrap(), INDEX_JS.len() as u64); - assert_eq!(resp.text()?, INDEX_JS); + assert!(resp.content_length().unwrap() > 10); + assert!(resp.text()?.contains("copyTextHandler")); Ok(()) }); @@ -190,8 +185,8 @@ mod tests { resp.headers().get("Content-Type"), Some(&"application/javascript".parse().unwrap()), ); - assert_eq!(resp.content_length().unwrap(), MENU_JS.len() as u64); - assert_eq!(resp.text()?, MENU_JS); + assert!(resp.content_length().unwrap() > 10); + assert!(resp.text()?.contains("closeMenu")); Ok(()) }); diff --git a/templates/index.js b/static/index.js similarity index 100% rename from templates/index.js rename to static/index.js diff --git a/templates/menu.js b/static/menu.js similarity index 100% rename from templates/menu.js rename to static/menu.js diff --git a/src/web/opensearch.xml b/static/opensearch.xml similarity index 91% rename from src/web/opensearch.xml rename to static/opensearch.xml index e12da194d..88d09f7e6 100644 --- a/src/web/opensearch.xml +++ b/static/opensearch.xml @@ -1,6 +1,6 @@ Docs.rs Search for crate documentation on docs.rs - https://docs.rs/favicon.ico + https://docs.rs/-/static/favicon.ico diff --git a/static/robots.txt b/static/robots.txt new file mode 100644 index 000000000..7dd3caf3e --- /dev/null +++ b/static/robots.txt @@ -0,0 +1 @@ +Sitemap: https://docs.rs/sitemap.xml diff --git a/templates/base.html b/templates/base.html index a8a61d433..92d335672 100644 --- a/templates/base.html +++ b/templates/base.html @@ -18,7 +18,7 @@ {%- block css -%}{%- endblock css -%} - + {%- block title -%} Docs.rs {%- endblock title -%} diff --git a/templates/rustdoc/head.html b/templates/rustdoc/head.html index 2b6cc3020..73181e0bd 100644 --- a/templates/rustdoc/head.html +++ b/templates/rustdoc/head.html @@ -1,4 +1,4 @@ {%- import "macros.html" as macros -%} - +