Skip to content

show pending CDN invalidations on queue page #1897

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 14, 2022
+199 −23
Merged

show pending CDN invalidations on queue page #1897

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7e74894
show pending CDN invalidations on queue page
syphar Nov 7, 2022
fbe9eab
fix typo in comment
syphar Nov 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 90 additions & 1 deletion src/cdn.rs
View file
Original file line number Diff line number Diff line change
@@ -4,6 +4,8 @@ use aws_sdk_cloudfront::{
model::{InvalidationBatch, Paths},
Client, RetryConfig,
};
use chrono::{DateTime, Utc};
use serde::Serialize;
use std::sync::{Arc, Mutex};
use strum::EnumString;
use tokio::runtime::Runtime;
@@ -133,14 +135,51 @@ pub(crate) fn invalidate_crate(config: &Config, cdn: &CdnBackend, name: &str) ->
Ok(())
}

#[derive(Debug, Clone, Serialize, PartialEq, Eq)]
pub(crate) struct CrateInvalidation {
pub name: String,
pub created: DateTime<Utc>,
}

/// Return fake active cloudfront invalidations.
/// CloudFront invalidations can take up to 15 minutes. Until we have
/// live queries of the invalidation status we just assume it's fine
/// 20 minutes after the build.
/// TODO: should be replaced be keeping track or querying the active invalidation from CloudFront
pub(crate) fn active_crate_invalidations(
conn: &mut postgres::Client,
) -> Result<Vec<CrateInvalidation>> {
Ok(conn
.query(
r#"
SELECT
crates.name,
MIN(builds.build_time) as build_time
FROM crates
INNER JOIN releases ON crates.id = releases.crate_id
INNER JOIN builds ON releases.id = builds.rid
WHERE builds.build_time >= CURRENT_TIMESTAMP - INTERVAL '20 minutes'
GROUP BY crates.name
ORDER BY MIN(builds.build_time)"#,
&[],
)?
.iter()
.map(|row| CrateInvalidation {
name: row.get(0),
created: row.get(1),
})
.collect())
}

#[cfg(test)]
mod tests {
use super::*;
use crate::test::wrapper;
use crate::test::{wrapper, FakeBuild};

use aws_sdk_cloudfront::{Client, Config, Credentials, Region};
use aws_smithy_client::{erase::DynConnector, test_connection::TestConnection};
use aws_smithy_http::body::SdkBody;
use chrono::{Duration, Timelike};

#[test]
fn create_cloudfront() {
@@ -213,6 +252,56 @@ mod tests {
Config::new(&cfg)
}

#[test]
fn get_active_invalidations() {
wrapper(|env| {
let now = Utc::now().with_nanosecond(0).unwrap();
let past_deploy = now - Duration::minutes(21);
let first_running_deploy = now - Duration::minutes(10);
let second_running_deploy = now;

env.fake_release()
.name("krate_2")
.version("0.0.1")
.builds(vec![FakeBuild::default().build_time(first_running_deploy)])
.create()?;

env.fake_release()
.name("krate_2")
.version("0.0.2")
.builds(vec![FakeBuild::default().build_time(second_running_deploy)])
.create()?;

env.fake_release()
.name("krate_1")
.version("0.0.2")
.builds(vec![FakeBuild::default().build_time(second_running_deploy)])
.create()?;

env.fake_release()
.name("krate_1")
.version("0.0.3")
.builds(vec![FakeBuild::default().build_time(past_deploy)])
.create()?;

assert_eq!(
active_crate_invalidations(&mut env.db().conn())?,
vec![
CrateInvalidation {
name: "krate_2".into(),
created: first_running_deploy,
},
CrateInvalidation {
name: "krate_1".into(),
created: second_running_deploy,
}
]
);

Ok(())
})
}

#[tokio::test]
async fn invalidate_path() {
let conn = TestConnection::new(vec![(
15 changes: 15 additions & 0 deletions src/test/fakes.rs
View file
Original file line number Diff line number Diff line change
@@ -38,6 +38,7 @@ pub(crate) struct FakeRelease<'a> {
pub(crate) struct FakeBuild {
s3_build_log: Option<String>,
db_build_log: Option<String>,
build_time: Option<DateTime<Utc>>,
result: BuildResult,
}

@@ -458,6 +459,12 @@ impl FakeGithubStats {
}

impl FakeBuild {
pub(crate) fn build_time(self, build_time: impl Into<DateTime<Utc>>) -> Self {
Self {
build_time: Some(build_time.into()),
..self
}
}
pub(crate) fn rustc_version(self, rustc_version: impl Into<String>) -> Self {
Self {
result: BuildResult {
@@ -525,6 +532,13 @@ impl FakeBuild {
)?;
}

if let Some(build_time) = self.build_time.as_ref() {
conn.query(
"UPDATE builds SET build_time = $2 WHERE id = $1",
&[&build_id, &build_time],
)?;
}

if let Some(s3_build_log) = self.s3_build_log.as_deref() {
let path = format!("build-logs/{}/{}.txt", build_id, default_target);
storage.store_one(path, s3_build_log)?;
@@ -539,6 +553,7 @@ impl Default for FakeBuild {
Self {
s3_build_log: Some("It works!".into()),
db_build_log: None,
build_time: None,
result: BuildResult {
rustc_version: "rustc 2.0.0-nightly (000000000 1970-01-01)".into(),
docsrs_version: "docs.rs 1.0.0 (000000000 1970-01-01)".into(),
51 changes: 48 additions & 3 deletions src/web/releases.rs
View file
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@

use crate::{
build_queue::QueuedCrate,
cdn::{self, CrateInvalidation},
db::{Pool, PoolClient},
impl_webpage,
utils::report_error,
@@ -668,6 +669,7 @@ pub fn activity_handler(req: &mut Request) -> IronResult<Response> {
struct BuildQueuePage {
description: &'static str,
queue: Vec<QueuedCrate>,
active_deployments: Vec<CrateInvalidation>,
}

impl_webpage! {
@@ -683,9 +685,12 @@ pub fn build_queue_handler(req: &mut Request) -> IronResult<Response> {
krate.priority = -krate.priority;
}

let mut conn = extension!(req, Pool).get()?;

BuildQueuePage {
description: "List of crates scheduled to build",
description: "crate documentation scheduled to build & deploy",
queue,
active_deployments: ctry!(req, cdn::active_crate_invalidations(&mut conn)),
}
.into_response(req)
}
@@ -695,7 +700,8 @@ mod tests {
use super::*;
use crate::index::api::CrateOwner;
use crate::test::{
assert_redirect, assert_redirect_unchecked, assert_success, wrapper, TestFrontend,
assert_redirect, assert_redirect_unchecked, assert_success, wrapper, FakeBuild,
TestFrontend,
};
use anyhow::Error;
use chrono::{Duration, TimeZone};
@@ -1326,6 +1332,40 @@ mod tests {
})
}

#[test]
fn test_deployment_queue() {
wrapper(|env| {
let web = env.frontend();

env.fake_release()
.name("krate_2")
.version("0.0.1")
.builds(vec![
FakeBuild::default().build_time(Utc::now() - Duration::minutes(10))
])
.create()?;

let empty = kuchiki::parse_html().one(web.get("/releases/queue").send()?.text()?);
assert!(empty
.select(".release > strong")
.expect("missing heading")
.any(|el| el.text_contents().contains("active CDN deployments")));

let full = kuchiki::parse_html().one(web.get("/releases/queue").send()?.text()?);
let items = full
.select(".queue-list > li")
.expect("missing list items")
.collect::<Vec<_>>();

assert_eq!(items.len(), 1);
let a = items[0].as_node().select_first("a").expect("missing link");

assert!(a.text_contents().contains("krate_2"));

Ok(())
});
}

#[test]
fn test_releases_queue() {
wrapper(|env| {
@@ -1334,10 +1374,15 @@ mod tests {

let empty = kuchiki::parse_html().one(web.get("/releases/queue").send()?.text()?);
assert!(empty
.select(".release > strong")
.select(".queue-list > strong")
.expect("missing heading")
.any(|el| el.text_contents().contains("nothing")));

assert!(!empty
.select(".release > strong")
.expect("missing heading")
.any(|el| el.text_contents().contains("active CDN deployments")));

queue.add_crate("foo", "1.0.0", 0, None)?;
queue.add_crate("bar", "0.1.0", -10, None)?;
queue.add_crate("baz", "0.0.1", 10, None)?;
2 changes: 1 addition & 1 deletion templates/footer.html
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<div class="docs-rs-footer">
<a href="/about">About docs.rs</a>
<a href="https://foundation.rust-lang.org/policies/privacy-policy/#docs.rs">Privacy policy</a>
<a href="/releases/queue">Build queue</a>
<a href="/releases/queue">Queue</a>
</div>
63 changes: 45 additions & 18 deletions templates/releases/build_queue.html
View file
Original file line number Diff line number Diff line change
@@ -1,37 +1,64 @@
{%- extends "base.html" -%}
{%- import "releases/header.html" as release_macros -%}

{%- block title -%}Build Queue - Docs.rs{%- endblock title -%}
{%- block title -%}Queue - Docs.rs{%- endblock title -%}

{%- block header -%}
{{ release_macros::header(title="Build Queue", description=description, tab="queue") }}
{{ release_macros::header(title="Queue", description=description, tab="queue") }}
{%- endblock header -%}

{%- block body -%}
<div class="container">
<div class="recent-releases-container">
{%- if active_deployments %}
<div class="release">
<strong>active CDN deployments</strong>
</div>

<div class = "pure-g">
<div class="pure-u-1-2">
<ol class="queue-list">
{% for invalidation in active_deployments -%}
<li>
<a href="https://docs.rs/{{ invalidation.name }}">
{{ invalidation.name }}
</a>
</li>
{%- endfor %}
</ol>
</div>
<div class="pure-u-1-2">
<div class="about">
<p>
After the build finishes it may take up to 20 minutes for all documentation
pages to be up-to-date and available to everybody.
</p>
<p>Especially <code>/latest/</code> URLs might be affected.</p>
</div>
</div>
</div>
{%- endif %}

<div class="release">
{% set queue_length = queue | length -%}
{%- if queue_length == 0 -%}
<strong>There is nothing in the queue</strong>
{%- else -%}
<strong>Queue</strong>
{%- endif %}
<strong>Build Queue</strong>
</div>

<ol class="queue-list">
{% for crate in queue -%}
<li>
<a href="https://crates.io/crates/{{ crate.name }}">
{{ crate.name }} {{ crate.version }}
</a>
{%- if queue -%}
{% for crate in queue -%}
<li>
<a href="https://crates.io/crates/{{ crate.name }}">
{{ crate.name }} {{ crate.version }}
</a>

{% if crate.priority != 0 -%}
(priority: {{ crate.priority }})
{%- endif %}
</li>
{%- endfor %}
{% if crate.priority != 0 -%}
(priority: {{ crate.priority }})
{%- endif %}
</li>
{%- endfor %}
{%- else %}
<strong>There is nothing in the queue</strong>
{%- endif %}
</ol>
</div>
</div>