Invalidate CDN caches after uploading database dump

src/background_jobs.rs

@@ -12,6 +12,7 @@ use crate::swirl::PerformError;
 use crate::uploaders::Uploader;
 use crate::worker;
 use crate::worker::cloudfront::CloudFront;
+use crate::worker::fastly::Fastly;
 use crates_io_index::Repository;
 
 pub const PRIORITY_DEFAULT: i16 = 0;
@@ -295,6 +296,7 @@ pub struct Environment {
     pub uploader: Uploader,
     http_client: AssertUnwindSafe<Client>,
     cloudfront: Option<CloudFront>,
+    fastly: Option<Fastly>,
 }
 
 impl Clone for Environment {
@@ -304,6 +306,7 @@ impl Clone for Environment {
             uploader: self.uploader.clone(),
             http_client: AssertUnwindSafe(self.http_client.0.clone()),
             cloudfront: self.cloudfront.clone(),
+            fastly: self.fastly.clone(),
         }
     }
 }
@@ -314,12 +317,14 @@ impl Environment {
         uploader: Uploader,
         http_client: Client,
         cloudfront: Option<CloudFront>,
+        fastly: Option<Fastly>,
     ) -> Self {
         Self::new_shared(
             Arc::new(Mutex::new(index)),
             uploader,
             http_client,
             cloudfront,
+            fastly,
         )
     }
 
@@ -328,12 +333,14 @@ impl Environment {
         uploader: Uploader,
         http_client: Client,
         cloudfront: Option<CloudFront>,
+        fastly: Option<Fastly>,
     ) -> Self {
         Self {
             index,
             uploader,
             http_client: AssertUnwindSafe(http_client),
             cloudfront,
+            fastly,
         }
     }
 
@@ -352,4 +359,8 @@ impl Environment {
     pub(crate) fn cloudfront(&self) -> Option<&CloudFront> {
         self.cloudfront.as_ref()
     }
+
+    pub(crate) fn fastly(&self) -> Option<&Fastly> {
+        self.fastly.as_ref()
+    }
 }

src/bin/background-worker.rs

@@ -26,6 +26,7 @@ use std::thread::sleep;
 use std::time::{Duration, Instant};
 
 use crates_io::swirl;
+use crates_io::worker::fastly::Fastly;
 
 fn main() {
     let _sentry = crates_io::sentry::init();
@@ -73,6 +74,7 @@ fn main() {
     info!(duration = ?clone_duration, "Index cloned");
 
     let cloudfront = CloudFront::from_environment();
+    let fastly = Fastly::from_environment();
 
     let build_runner = || {
         let client = Client::builder()
@@ -84,6 +86,7 @@ fn main() {
             uploader.clone(),
             client,
             cloudfront.clone(),
+            fastly.clone(),
         );
         swirl::Runner::production_runner(environment, db_url.clone(), job_start_timeout)
     };

src/worker/dump_db.rs

@@ -30,6 +30,10 @@ pub fn perform_dump_db(
     info!("Uploading tarball");
     let size = tarball.upload(&target_name, &env.uploader)?;
     info!("Database dump uploaded {} bytes to {}.", size, &target_name);
+
+    info!("Invalidating CDN caches");
+    invalidate_caches(env, &target_name)?;
+
     Ok(())
 }
 
@@ -246,6 +250,22 @@ impl Drop for DumpTarball {
     }
 }
 
+fn invalidate_caches(env: &Environment, target_name: &str) -> Result<(), PerformError> {
+    if let Some(cloudfront) = env.cloudfront() {
+        if let Err(error) = cloudfront.invalidate(env.http_client(), target_name) {
+            warn!("failed to invalidate CloudFront cache: {}", error);
+        }
+    }
+
+    if let Some(fastly) = env.fastly() {
+        if let Err(error) = fastly.invalidate(env.http_client(), target_name) {
+            warn!("failed to invalidate Fastly cache: {}", error);
+        }
+    }
+
+    Ok(())
+}
+
 mod configuration;
 mod gen_scripts;
 

src/worker/fastly.rs

@@ -0,0 +1,77 @@
+use anyhow::{anyhow, Context};
+use reqwest::blocking::Client;
+
+#[derive(Clone, Debug)]
+pub struct Fastly {
+    api_token: String,
+    static_domain_name: String,
+}
+
+impl Fastly {
+    pub fn from_environment() -> Option<Self> {
+        let api_token = dotenvy::var("FASTLY_API_TOKEN").ok()?;
+        let static_domain_name = dotenvy::var("S3_CDN").expect("missing S3_CDN");
+
+        Some(Self {
+            api_token,
+            static_domain_name,
+        })
+    }
+
+    /// Invalidate a path on Fastly
+    ///
+    /// This method takes a path and invalidates the cached content on Fastly. The path must not
+    /// contain a wildcard, since the Fastly API does not support wildcard invalidations.
+    ///
+    /// Requests are authenticated using a token that is sent in a header. The token is passed to
+    /// the application as an environment variable.
+    ///
+    /// More information on Fastly's APIs for cache invalidations can be found here:
+    /// https://developer.fastly.com/reference/api/purging/
+    #[instrument(skip(self, client))]
+    pub fn invalidate(&self, client: &Client, path: &str) -> anyhow::Result<()> {
+        if path.contains('*') {
+            return Err(anyhow!(
+                "wildcard invalidations are not supported for Fastly"
+            ));
+        }
+
+        let path = path.trim_start_matches('/');
+        let url = format!(
+            "https://api.fastly.com/purge/{}/{}",
+            self.static_domain_name, path
+        );
+        trace!(?url);
+
+        let mut headers = reqwest::header::HeaderMap::new();
+        headers.append("Fastly-Key", self.api_token.parse()?);
+
+        debug!("sending invalidation request to Fastly");
+        let response = client
+            .post(&url)
+            .headers(headers)
+            .send()
+            .context("failed to send invalidation request to Fastly")?;
+
+        let status = response.status();
+
+        match response.error_for_status_ref() {
+            Ok(_) => {
+                debug!(?status, "invalidation request accepted by Fastly");
+                Ok(())
+            }
+            Err(error) => {
+                let headers = response.headers().clone();
+                let body = response.text();
+                warn!(
+                    ?status,
+                    ?headers,
+                    ?body,
+                    "invalidation request to Fastly failed"
+                );
+
+                Err(error).with_context(|| format!("failed to invalidate {path} on Fastly"))
+            }
+        }
+    }
+}

src/worker/mod.rs

@@ -6,6 +6,7 @@
 pub mod cloudfront;
 mod daily_db_maintenance;
 pub mod dump_db;
+pub mod fastly;
 mod git;
 mod readmes;
 mod update_downloads;