Add metrics tracking number of recently loaded crates/versions/platforms

Cargo.toml

@@ -51,6 +51,8 @@ base64 = "0.12.1"
 strum = { version = "0.18.0", features = ["derive"] }
 lol_html = "0.2"
 font-awesome-as-a-crate = { path = "crates/font-awesome-as-a-crate" }
+dashmap = "3.11.10"
+string_cache = "0.8.0"
 
 # Async
 tokio = { version = "0.2.22", features = ["rt-threaded"] }
@@ -97,6 +99,7 @@ rand = "0.7.3"
 time = "0.1"
 git2 = { version = "0.13", default-features = false }
 sass-rs = "0.2.2"
+string_cache_codegen = "0.5.1"
 
 [[bench]]
 name = "html_parsing"

build.rs

@@ -19,6 +19,7 @@ fn main() {
     if let Err(sass_err) = compile_sass() {
         panic!("Error compiling sass: {}", sass_err);
     }
+    write_known_targets().unwrap();
 }
 
 fn write_git_version() {
@@ -87,3 +88,21 @@ fn compile_sass() -> Result<(), Box<dyn Error>> {
 
     Ok(())
 }
+
+fn write_known_targets() -> std::io::Result<()> {
+    use std::io::BufRead;
+
+    let targets: Vec<String> = std::process::Command::new("rustc")
+        .args(&["--print", "target-list"])
+        .output()?
+        .stdout
+        .lines()
+        .filter(|s| s.as_ref().map_or(true, |s| !s.is_empty()))
+        .collect::<std::io::Result<_>>()?;
+
+    string_cache_codegen::AtomType::new("target::TargetAtom", "target_atom!")
+        .atoms(&targets)
+        .write_to_file(&Path::new(&env::var("OUT_DIR").unwrap()).join("target_atom.rs"))?;
+
+    Ok(())
+}

src/lib.rs

@@ -26,6 +26,13 @@ mod test;
 pub mod utils;
 mod web;
 
+#[allow(dead_code)]
+mod target {
+    //! [`crate::target::TargetAtom`] is an interned string type for rustc targets, such as
+    //! `x86_64-unknown-linux-gnu`. See the [`string_cache`] docs for usage examples.
+    include!(concat!(env!("OUT_DIR"), "/target_atom.rs"));
+}
+
 use web::page::GlobalAlert;
 
 // Warning message shown in the navigation bar of every page. Set to `None` to hide it.

src/metrics/macros.rs

@@ -20,6 +20,7 @@ macro_rules! metrics {
                 $(#[$meta])*
                 $metric_vis $metric: $ty,
             )*
+            pub(crate) recently_accessed_releases: RecentlyAccessedReleases,
         }
         impl $name {
             $vis fn new() -> Result<Self, prometheus::Error> {
@@ -36,6 +37,7 @@ macro_rules! metrics {
                 )*
                 Ok(Self {
                     registry,
+                    recently_accessed_releases: RecentlyAccessedReleases::new(),
                     $(
                         $(#[$meta])*
                         $metric,

src/metrics/mod.rs

@@ -3,13 +3,17 @@ mod macros;
 
 use self::macros::MetricFromOpts;
 use crate::db::Pool;
+use crate::target::TargetAtom;
 use crate::BuildQueue;
+use dashmap::DashMap;
 use failure::Error;
 use prometheus::proto::MetricFamily;
+use std::time::{Duration, Instant};
 
 load_metric_type!(IntGauge as single);
 load_metric_type!(IntCounter as single);
 load_metric_type!(IntCounterVec as vec);
+load_metric_type!(IntGaugeVec as vec);
 load_metric_type!(HistogramVec as vec);
 
 metrics! {
@@ -44,6 +48,13 @@ metrics! {
         /// The time it takes to render a rustdoc page
         pub(crate) rustdoc_rendering_times: HistogramVec["step"],
 
+        /// Count of recently accessed crates
+        pub(crate) recent_crates: IntGaugeVec["duration"],
+        /// Count of recently accessed versions of crates
+        pub(crate) recent_versions: IntGaugeVec["duration"],
+        /// Count of recently accessed platforms of versions of crates
+        pub(crate) recent_platforms: IntGaugeVec["duration"],
+
         /// Number of crates built
         pub(crate) total_builds: IntCounter,
         /// Number of builds that successfully generated docs
@@ -67,6 +78,70 @@ metrics! {
     namespace: "docsrs",
 }
 
+#[derive(Debug, Default)]
+pub(crate) struct RecentlyAccessedReleases {
+    crates: DashMap<i32, Instant>,
+    versions: DashMap<i32, Instant>,
+    platforms: DashMap<(i32, TargetAtom), Instant>,
+}
+
+impl RecentlyAccessedReleases {
+    pub(crate) fn new() -> Self {
+        Self::default()
+    }
+
+    pub(crate) fn record(&self, krate: i32, version: i32, target: &str) {
+        if self.platforms.len() > 100_000 {
+            // Avoid filling the maps _too_ much, we should never get anywhere near this limit
+            return;
+        }
+
+        let now = Instant::now();
+        self.crates.insert(krate, now);
+        self.versions.insert(version, now);
+        self.platforms
+            .insert((version, TargetAtom::from(target)), now);
+    }
+
+    pub(crate) fn gather(&self, metrics: &Metrics) {
+        fn inner<K: std::hash::Hash + Eq>(map: &DashMap<K, Instant>, metric: &IntGaugeVec) {
+            let mut hour_count = 0;
+            let mut half_hour_count = 0;
+            let mut five_minute_count = 0;
+            map.retain(|_, instant| {
+                let elapsed = instant.elapsed();
+
+                if elapsed < Duration::from_secs(60 * 60) {
+                    hour_count += 1;
+                }
+                if elapsed < Duration::from_secs(30 * 60) {
+                    half_hour_count += 1;
+                }
+                if elapsed < Duration::from_secs(5 * 60) {
+                    five_minute_count += 1;
+                }
+
+                // Only retain items accessed within the last hour
+                elapsed < Duration::from_secs(60 * 60)
+            });
+
+            metric.with_label_values(&["one hour"]).set(hour_count);
+
+            metric
+                .with_label_values(&["half hour"])
+                .set(half_hour_count);
+
+            metric
+                .with_label_values(&["five minutes"])
+                .set(five_minute_count);
+        }
+
+        inner(&self.crates, &metrics.recent_crates);
+        inner(&self.versions, &metrics.recent_versions);
+        inner(&self.platforms, &metrics.recent_platforms);
+    }
+}
+
 impl Metrics {
     pub(crate) fn gather(
         &self,
@@ -82,6 +157,7 @@ impl Metrics {
             .set(queue.prioritized_count()? as i64);
         self.failed_crates_count.set(queue.failed_count()? as i64);
 
+        self.recently_accessed_releases.gather(self);
         self.gather_system_performance();
         Ok(self.registry.gather())
     }

src/web/crate_details.rs

@@ -45,6 +45,10 @@ pub struct CrateDetails {
     documented_items: Option<f32>,
     total_items_needing_examples: Option<f32>,
     items_with_examples: Option<f32>,
+    /// Database id for this crate
+    pub(crate) crate_id: i32,
+    /// Database id for this release
+    pub(crate) release_id: i32,
 }
 
 fn optional_markdown<S>(markdown: &Option<String>, serializer: S) -> Result<S::Ok, S::Error>
@@ -183,6 +187,8 @@ impl CrateDetails {
             total_items: total_items.map(|v| v as f32),
             total_items_needing_examples: total_items_needing_examples.map(|v| v as f32),
             items_with_examples: items_with_examples.map(|v| v as f32),
+            crate_id,
+            release_id,
         };
 
         if let Some(repository_url) = crate_details.repository_url.clone() {

src/web/rustdoc.rs

@@ -415,16 +415,24 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult<Response> {
                 .iter()
                 .any(|s| s == inner_path[0])
         {
-            let mut target = inner_path.remove(0).to_string();
-            target.push('/');
-            target
+            inner_path.remove(0)
         } else {
-            String::new()
+            ""
         };
 
         (target, inner_path.join("/"))
     };
 
+    metrics
+        .recently_accessed_releases
+        .record(krate.crate_id, krate.release_id, target);
+
+    let target = if target == "" {
+        String::new()
+    } else {
+        format!("{}/", target)
+    };
+
     rendering_time.step("rewrite html");
     RustdocPage {
         latest_path,