diff --git a/Cargo.lock b/Cargo.lock index 3ca1fe649..11536083f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,6 +21,15 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" +dependencies = [ + "const-random", +] + [[package]] name = "aho-corasick" version = "0.7.10" @@ -325,6 +334,26 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "const-random" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a" +dependencies = [ + "const-random-macro", + "proc-macro-hack", +] + +[[package]] +name = "const-random-macro" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a" +dependencies = [ + "getrandom", + "proc-macro-hack", +] + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -564,6 +593,17 @@ dependencies = [ "memchr 2.3.3", ] +[[package]] +name = "dashmap" +version = "3.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5" +dependencies = [ + "ahash", + "cfg-if", + "num_cpus", +] + [[package]] name = "derive_more" version = "0.99.7" @@ -633,6 +673,7 @@ dependencies = [ "crates-index", "crates-index-diff", "criterion", + "dashmap", "docsrs-metadata", "dotenv", "env_logger", @@ -668,6 +709,8 @@ dependencies = [ "serde", "serde_json", "slug", + "string_cache", + "string_cache_codegen", "structopt", "strum", "systemstat", diff --git a/Cargo.toml b/Cargo.toml index 872a05636..97af2a5da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,8 @@ base64 = "0.12.1" strum = { version = "0.18.0", features = ["derive"] } lol_html = "0.2" font-awesome-as-a-crate = { path = "crates/font-awesome-as-a-crate" } +dashmap = "3.11.10" +string_cache = "0.8.0" # Async tokio = { version = "0.2.22", features = ["rt-threaded"] } @@ -97,6 +99,7 @@ rand = "0.7.3" time = "0.1" git2 = { version = "0.13", default-features = false } sass-rs = "0.2.2" +string_cache_codegen = "0.5.1" [[bench]] name = "html_parsing" diff --git a/build.rs b/build.rs index 0e2e5d54d..323e690d9 100644 --- a/build.rs +++ b/build.rs @@ -19,6 +19,7 @@ fn main() { if let Err(sass_err) = compile_sass() { panic!("Error compiling sass: {}", sass_err); } + write_known_targets().unwrap(); } fn write_git_version() { @@ -87,3 +88,21 @@ fn compile_sass() -> Result<(), Box> { Ok(()) } + +fn write_known_targets() -> std::io::Result<()> { + use std::io::BufRead; + + let targets: Vec = std::process::Command::new("rustc") + .args(&["--print", "target-list"]) + .output()? + .stdout + .lines() + .filter(|s| s.as_ref().map_or(true, |s| !s.is_empty())) + .collect::>()?; + + string_cache_codegen::AtomType::new("target::TargetAtom", "target_atom!") + .atoms(&targets) + .write_to_file(&Path::new(&env::var("OUT_DIR").unwrap()).join("target_atom.rs"))?; + + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 5c3f06eea..c129cda08 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,13 @@ mod test; pub mod utils; mod web; +#[allow(dead_code)] +mod target { + //! [`crate::target::TargetAtom`] is an interned string type for rustc targets, such as + //! `x86_64-unknown-linux-gnu`. See the [`string_cache`] docs for usage examples. + include!(concat!(env!("OUT_DIR"), "/target_atom.rs")); +} + use web::page::GlobalAlert; // Warning message shown in the navigation bar of every page. Set to `None` to hide it. diff --git a/src/metrics/macros.rs b/src/metrics/macros.rs index cf7ccd256..22fc97be7 100644 --- a/src/metrics/macros.rs +++ b/src/metrics/macros.rs @@ -20,6 +20,7 @@ macro_rules! metrics { $(#[$meta])* $metric_vis $metric: $ty, )* + pub(crate) recently_accessed_releases: RecentlyAccessedReleases, } impl $name { $vis fn new() -> Result { @@ -36,6 +37,7 @@ macro_rules! metrics { )* Ok(Self { registry, + recently_accessed_releases: RecentlyAccessedReleases::new(), $( $(#[$meta])* $metric, diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 9f5eabebd..f5142035e 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -3,13 +3,17 @@ mod macros; use self::macros::MetricFromOpts; use crate::db::Pool; +use crate::target::TargetAtom; use crate::BuildQueue; +use dashmap::DashMap; use failure::Error; use prometheus::proto::MetricFamily; +use std::time::{Duration, Instant}; load_metric_type!(IntGauge as single); load_metric_type!(IntCounter as single); load_metric_type!(IntCounterVec as vec); +load_metric_type!(IntGaugeVec as vec); load_metric_type!(HistogramVec as vec); metrics! { @@ -44,6 +48,13 @@ metrics! { /// The time it takes to render a rustdoc page pub(crate) rustdoc_rendering_times: HistogramVec["step"], + /// Count of recently accessed crates + pub(crate) recent_crates: IntGaugeVec["duration"], + /// Count of recently accessed versions of crates + pub(crate) recent_versions: IntGaugeVec["duration"], + /// Count of recently accessed platforms of versions of crates + pub(crate) recent_platforms: IntGaugeVec["duration"], + /// Number of crates built pub(crate) total_builds: IntCounter, /// Number of builds that successfully generated docs @@ -67,6 +78,70 @@ metrics! { namespace: "docsrs", } +#[derive(Debug, Default)] +pub(crate) struct RecentlyAccessedReleases { + crates: DashMap, + versions: DashMap, + platforms: DashMap<(i32, TargetAtom), Instant>, +} + +impl RecentlyAccessedReleases { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn record(&self, krate: i32, version: i32, target: &str) { + if self.platforms.len() > 100_000 { + // Avoid filling the maps _too_ much, we should never get anywhere near this limit + return; + } + + let now = Instant::now(); + self.crates.insert(krate, now); + self.versions.insert(version, now); + self.platforms + .insert((version, TargetAtom::from(target)), now); + } + + pub(crate) fn gather(&self, metrics: &Metrics) { + fn inner(map: &DashMap, metric: &IntGaugeVec) { + let mut hour_count = 0; + let mut half_hour_count = 0; + let mut five_minute_count = 0; + map.retain(|_, instant| { + let elapsed = instant.elapsed(); + + if elapsed < Duration::from_secs(60 * 60) { + hour_count += 1; + } + if elapsed < Duration::from_secs(30 * 60) { + half_hour_count += 1; + } + if elapsed < Duration::from_secs(5 * 60) { + five_minute_count += 1; + } + + // Only retain items accessed within the last hour + elapsed < Duration::from_secs(60 * 60) + }); + + metric.with_label_values(&["one hour"]).set(hour_count); + + metric + .with_label_values(&["half hour"]) + .set(half_hour_count); + + metric + .with_label_values(&["five minutes"]) + .set(five_minute_count); + } + + inner(&self.crates, &metrics.recent_crates); + inner(&self.versions, &metrics.recent_versions); + inner(&self.platforms, &metrics.recent_platforms); + } +} + impl Metrics { pub(crate) fn gather( &self, @@ -82,6 +157,7 @@ impl Metrics { .set(queue.prioritized_count()? as i64); self.failed_crates_count.set(queue.failed_count()? as i64); + self.recently_accessed_releases.gather(self); self.gather_system_performance(); Ok(self.registry.gather()) } diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index 15cc60ba8..9dfae8930 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -45,6 +45,10 @@ pub struct CrateDetails { documented_items: Option, total_items_needing_examples: Option, items_with_examples: Option, + /// Database id for this crate + pub(crate) crate_id: i32, + /// Database id for this release + pub(crate) release_id: i32, } fn optional_markdown(markdown: &Option, serializer: S) -> Result @@ -183,6 +187,8 @@ impl CrateDetails { total_items: total_items.map(|v| v as f32), total_items_needing_examples: total_items_needing_examples.map(|v| v as f32), items_with_examples: items_with_examples.map(|v| v as f32), + crate_id, + release_id, }; if let Some(repository_url) = crate_details.repository_url.clone() { diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 99a32b829..54da9e630 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -415,16 +415,24 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { .iter() .any(|s| s == inner_path[0]) { - let mut target = inner_path.remove(0).to_string(); - target.push('/'); - target + inner_path.remove(0) } else { - String::new() + "" }; (target, inner_path.join("/")) }; + metrics + .recently_accessed_releases + .record(krate.crate_id, krate.release_id, target); + + let target = if target == "" { + String::new() + } else { + format!("{}/", target) + }; + rendering_time.step("rewrite html"); RustdocPage { latest_path,