From 7e7489415e6cad19efd74b221c8a0878db3c73c2 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Mon, 7 Nov 2022 15:46:40 +0100 Subject: [PATCH 1/2] show pending CDN invalidations on queue page --- src/cdn.rs | 91 ++++++++++++++++++++++++++++- src/test/fakes.rs | 15 +++++ src/web/releases.rs | 51 +++++++++++++++- templates/footer.html | 2 +- templates/releases/build_queue.html | 63 ++++++++++++++------ 5 files changed, 199 insertions(+), 23 deletions(-) diff --git a/src/cdn.rs b/src/cdn.rs index 25235dd58..33dca4cb9 100644 --- a/src/cdn.rs +++ b/src/cdn.rs @@ -4,6 +4,8 @@ use aws_sdk_cloudfront::{ model::{InvalidationBatch, Paths}, Client, RetryConfig, }; +use chrono::{DateTime, Utc}; +use serde::Serialize; use std::sync::{Arc, Mutex}; use strum::EnumString; use tokio::runtime::Runtime; @@ -133,14 +135,51 @@ pub(crate) fn invalidate_crate(config: &Config, cdn: &CdnBackend, name: &str) -> Ok(()) } +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub(crate) struct CrateInvalidation { + pub name: String, + pub created: DateTime, +} + +/// Return fake active cloudfront invalidations. +/// CloudFront invalidations can take up to 15 minutes. Until we have +/// live queries of the invalidation status we just assume it's fine +/// latest 20 minutes after the build. +/// TODO: should be replaced be keeping track or querying the active invalidation from CloudFront +pub(crate) fn active_crate_invalidations( + conn: &mut postgres::Client, +) -> Result> { + Ok(conn + .query( + r#" + SELECT + crates.name, + MIN(builds.build_time) as build_time + FROM crates + INNER JOIN releases ON crates.id = releases.crate_id + INNER JOIN builds ON releases.id = builds.rid + WHERE builds.build_time >= CURRENT_TIMESTAMP - INTERVAL '20 minutes' + GROUP BY crates.name + ORDER BY MIN(builds.build_time)"#, + &[], + )? + .iter() + .map(|row| CrateInvalidation { + name: row.get(0), + created: row.get(1), + }) + .collect()) +} + #[cfg(test)] mod tests { use super::*; - use crate::test::wrapper; + use crate::test::{wrapper, FakeBuild}; use aws_sdk_cloudfront::{Client, Config, Credentials, Region}; use aws_smithy_client::{erase::DynConnector, test_connection::TestConnection}; use aws_smithy_http::body::SdkBody; + use chrono::{Duration, Timelike}; #[test] fn create_cloudfront() { @@ -213,6 +252,56 @@ mod tests { Config::new(&cfg) } + #[test] + fn get_active_invalidations() { + wrapper(|env| { + let now = Utc::now().with_nanosecond(0).unwrap(); + let past_deploy = now - Duration::minutes(21); + let first_running_deploy = now - Duration::minutes(10); + let second_running_deploy = now; + + env.fake_release() + .name("krate_2") + .version("0.0.1") + .builds(vec![FakeBuild::default().build_time(first_running_deploy)]) + .create()?; + + env.fake_release() + .name("krate_2") + .version("0.0.2") + .builds(vec![FakeBuild::default().build_time(second_running_deploy)]) + .create()?; + + env.fake_release() + .name("krate_1") + .version("0.0.2") + .builds(vec![FakeBuild::default().build_time(second_running_deploy)]) + .create()?; + + env.fake_release() + .name("krate_1") + .version("0.0.3") + .builds(vec![FakeBuild::default().build_time(past_deploy)]) + .create()?; + + assert_eq!( + active_crate_invalidations(&mut env.db().conn())?, + vec![ + CrateInvalidation { + name: "krate_2".into(), + created: first_running_deploy, + }, + CrateInvalidation { + name: "krate_1".into(), + created: second_running_deploy, + } + ] + ); + + Ok(()) + }) + } + #[tokio::test] async fn invalidate_path() { let conn = TestConnection::new(vec![( diff --git a/src/test/fakes.rs b/src/test/fakes.rs index dc5252626..fb3b98683 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -38,6 +38,7 @@ pub(crate) struct FakeRelease<'a> { pub(crate) struct FakeBuild { s3_build_log: Option, db_build_log: Option, + build_time: Option>, result: BuildResult, } @@ -458,6 +459,12 @@ impl FakeGithubStats { } impl FakeBuild { + pub(crate) fn build_time(self, build_time: impl Into>) -> Self { + Self { + build_time: Some(build_time.into()), + ..self + } + } pub(crate) fn rustc_version(self, rustc_version: impl Into) -> Self { Self { result: BuildResult { @@ -525,6 +532,13 @@ impl FakeBuild { )?; } + if let Some(build_time) = self.build_time.as_ref() { + conn.query( + "UPDATE builds SET build_time = $2 WHERE id = $1", + &[&build_id, &build_time], + )?; + } + if let Some(s3_build_log) = self.s3_build_log.as_deref() { let path = format!("build-logs/{}/{}.txt", build_id, default_target); storage.store_one(path, s3_build_log)?; @@ -539,6 +553,7 @@ impl Default for FakeBuild { Self { s3_build_log: Some("It works!".into()), db_build_log: None, + build_time: None, result: BuildResult { rustc_version: "rustc 2.0.0-nightly (000000000 1970-01-01)".into(), docsrs_version: "docs.rs 1.0.0 (000000000 1970-01-01)".into(), diff --git a/src/web/releases.rs b/src/web/releases.rs index db7463645..eda498a89 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -2,6 +2,7 @@ use crate::{ build_queue::QueuedCrate, + cdn::{self, CrateInvalidation}, db::{Pool, PoolClient}, impl_webpage, utils::report_error, @@ -668,6 +669,7 @@ pub fn activity_handler(req: &mut Request) -> IronResult { struct BuildQueuePage { description: &'static str, queue: Vec, + active_deployments: Vec, } impl_webpage! { @@ -683,9 +685,12 @@ pub fn build_queue_handler(req: &mut Request) -> IronResult { krate.priority = -krate.priority; } + let mut conn = extension!(req, Pool).get()?; + BuildQueuePage { - description: "List of crates scheduled to build", + description: "crate documentation scheduled to build & deploy", queue, + active_deployments: ctry!(req, cdn::active_crate_invalidations(&mut conn)), } .into_response(req) } @@ -695,7 +700,8 @@ mod tests { use super::*; use crate::index::api::CrateOwner; use crate::test::{ - assert_redirect, assert_redirect_unchecked, assert_success, wrapper, TestFrontend, + assert_redirect, assert_redirect_unchecked, assert_success, wrapper, FakeBuild, + TestFrontend, }; use anyhow::Error; use chrono::{Duration, TimeZone}; @@ -1326,6 +1332,40 @@ mod tests { }) } + #[test] + fn test_deployment_queue() { + wrapper(|env| { + let web = env.frontend(); + + env.fake_release() + .name("krate_2") + .version("0.0.1") + .builds(vec![ + FakeBuild::default().build_time(Utc::now() - Duration::minutes(10)) + ]) + .create()?; + + let empty = kuchiki::parse_html().one(web.get("/releases/queue").send()?.text()?); + assert!(empty + .select(".release > strong") + .expect("missing heading") + .any(|el| el.text_contents().contains("active CDN deployments"))); + + let full = kuchiki::parse_html().one(web.get("/releases/queue").send()?.text()?); + let items = full + .select(".queue-list > li") + .expect("missing list items") + .collect::>(); + + assert_eq!(items.len(), 1); + let a = items[0].as_node().select_first("a").expect("missing link"); + + assert!(a.text_contents().contains("krate_2")); + + Ok(()) + }); + } + #[test] fn test_releases_queue() { wrapper(|env| { @@ -1334,10 +1374,15 @@ mod tests { let empty = kuchiki::parse_html().one(web.get("/releases/queue").send()?.text()?); assert!(empty - .select(".release > strong") + .select(".queue-list > strong") .expect("missing heading") .any(|el| el.text_contents().contains("nothing"))); + assert!(!empty + .select(".release > strong") + .expect("missing heading") + .any(|el| el.text_contents().contains("active CDN deployments"))); + queue.add_crate("foo", "1.0.0", 0, None)?; queue.add_crate("bar", "0.1.0", -10, None)?; queue.add_crate("baz", "0.0.1", 10, None)?; diff --git a/templates/footer.html b/templates/footer.html index c92b076e4..d5a00ac28 100644 --- a/templates/footer.html +++ b/templates/footer.html @@ -1,5 +1,5 @@
About docs.rs Privacy policy - Build queue + Queue
diff --git a/templates/releases/build_queue.html b/templates/releases/build_queue.html index 37366aea9..1f0b8accb 100644 --- a/templates/releases/build_queue.html +++ b/templates/releases/build_queue.html @@ -1,37 +1,64 @@ {%- extends "base.html" -%} {%- import "releases/header.html" as release_macros -%} -{%- block title -%}Build Queue - Docs.rs{%- endblock title -%} +{%- block title -%}Queue - Docs.rs{%- endblock title -%} {%- block header -%} - {{ release_macros::header(title="Build Queue", description=description, tab="queue") }} + {{ release_macros::header(title="Queue", description=description, tab="queue") }} {%- endblock header -%} {%- block body -%}
+ {%- if active_deployments %} +
+ active CDN deployments +
+ +
+
+
    + {% for invalidation in active_deployments -%} +
  1. + + {{ invalidation.name }} + +
    2. + {%- endfor %} +
+
+
+
+

+ After the build finishes it may take up to 20 minutes for all documentation + pages to be up-to-date and available to everybody. +

+

Especially /latest/ URLs might be affected.

+
+
+
+ {%- endif %}
- {% set queue_length = queue | length -%} - {%- if queue_length == 0 -%} - There is nothing in the queue - {%- else -%} - Queue - {%- endif %} + Build Queue
    - {% for crate in queue -%} -
  1. - - {{ crate.name }} {{ crate.version }} - + {%- if queue -%} + {% for crate in queue -%} +
  2. + + {{ crate.name }} {{ crate.version }} + - {% if crate.priority != 0 -%} - (priority: {{ crate.priority }}) - {%- endif %} -
    3. - {%- endfor %} + {% if crate.priority != 0 -%} + (priority: {{ crate.priority }}) + {%- endif %} + + {%- endfor %} + {%- else %} + There is nothing in the queue + {%- endif %}
From fbe9eabfaa430397b58a07229d47387b17025eb5 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Mon, 14 Nov 2022 09:28:12 +0100 Subject: [PATCH 2/2] fix typo in comment --- src/cdn.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cdn.rs b/src/cdn.rs index 33dca4cb9..ab66d3a62 100644 --- a/src/cdn.rs +++ b/src/cdn.rs @@ -144,7 +144,7 @@ pub(crate) struct CrateInvalidation { /// Return fake active cloudfront invalidations. /// CloudFront invalidations can take up to 15 minutes. Until we have /// live queries of the invalidation status we just assume it's fine -/// latest 20 minutes after the build. +/// 20 minutes after the build. /// TODO: should be replaced be keeping track or querying the active invalidation from CloudFront pub(crate) fn active_crate_invalidations( conn: &mut postgres::Client,