diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 2bcc38bbd..dff84007e 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -5,7 +5,9 @@ use std::sync::Arc; use cratesfyi::db::{self, add_path_into_database, Pool}; use cratesfyi::utils::{remove_crate_priority, set_crate_priority}; -use cratesfyi::{BuildQueue, Config, DocBuilder, DocBuilderOptions, RustwideBuilder, Server}; +use cratesfyi::{ + BuildQueue, Config, DocBuilder, DocBuilderOptions, RustwideBuilder, Server, Storage, +}; use failure::{err_msg, Error, ResultExt}; use once_cell::sync::OnceCell; use structopt::StructOpt; @@ -123,6 +125,7 @@ impl CommandLine { ctx.pool()?, ctx.config()?, ctx.build_queue()?, + ctx.storage()?, )?; } Self::Daemon { @@ -137,6 +140,7 @@ impl CommandLine { ctx.config()?, ctx.pool()?, ctx.build_queue()?, + ctx.storage()?, registry_watcher == Toggle::Enabled, )?; } @@ -346,7 +350,7 @@ impl BuildSubcommand { Self::World => { docbuilder.load_cache().context("Failed to load cache")?; - let mut builder = RustwideBuilder::init(ctx.pool()?)?; + let mut builder = RustwideBuilder::init(ctx.pool()?, ctx.storage()?)?; builder .build_world(&mut docbuilder) .context("Failed to build world")?; @@ -360,8 +364,8 @@ impl BuildSubcommand { local, } => { docbuilder.load_cache().context("Failed to load cache")?; - let mut builder = - RustwideBuilder::init(ctx.pool()?).context("failed to initialize rustwide")?; + let mut builder = RustwideBuilder::init(ctx.pool()?, ctx.storage()?) + .context("failed to initialize rustwide")?; if let Some(path) = local { builder @@ -397,14 +401,14 @@ impl BuildSubcommand { } } - let mut builder = RustwideBuilder::init(ctx.pool()?)?; + let mut builder = RustwideBuilder::init(ctx.pool()?, ctx.storage()?)?; builder .update_toolchain() .context("failed to update toolchain")?; } Self::AddEssentialFiles => { - let mut builder = RustwideBuilder::init(ctx.pool()?)?; + let mut builder = RustwideBuilder::init(ctx.pool()?, ctx.storage()?)?; builder .add_essential_files() .context("failed to add essential files")?; @@ -469,7 +473,7 @@ impl DatabaseSubcommand { } Self::AddDirectory { directory, prefix } => { - add_path_into_database(&*ctx.conn()?, &prefix, directory) + add_path_into_database(&*ctx.storage()?, &prefix, directory) .context("Failed to add directory into database")?; } @@ -553,6 +557,7 @@ enum DeleteSubcommand { struct Context { build_queue: OnceCell>, + storage: OnceCell>, config: OnceCell>, pool: OnceCell, } @@ -561,6 +566,7 @@ impl Context { fn new() -> Self { Self { build_queue: OnceCell::new(), + storage: OnceCell::new(), config: OnceCell::new(), pool: OnceCell::new(), } @@ -575,6 +581,13 @@ impl Context { .clone()) } + fn storage(&self) -> Result, Error> { + Ok(self + .storage + .get_or_try_init::<_, Error>(|| Ok(Arc::new(Storage::new(self.pool()?))))? + .clone()) + } + fn config(&self) -> Result, Error> { Ok(self .config diff --git a/src/db/delete.rs b/src/db/delete.rs index 23dc22672..1804e97d3 100644 --- a/src/db/delete.rs +++ b/src/db/delete.rs @@ -190,17 +190,17 @@ mod tests { let db = env.db(); // Create fake packages in the database - let pkg1_v1_id = db + let pkg1_v1_id = env .fake_release() .name("package-1") .version("1.0.0") .create()?; - let pkg1_v2_id = db + let pkg1_v2_id = env .fake_release() .name("package-1") .version("2.0.0") .create()?; - let pkg2_id = db.fake_release().name("package-2").create()?; + let pkg2_id = env.fake_release().name("package-2").create()?; assert!(crate_exists(&db.conn(), "package-1")?); assert!(crate_exists(&db.conn(), "package-2")?); @@ -244,13 +244,13 @@ mod tests { } let db = env.db(); - let v1 = db + let v1 = env .fake_release() .name("a") .version("1.0.0") .author("malicious actor") .create()?; - let v2 = db + let v2 = env .fake_release() .name("a") .version("2.0.0") diff --git a/src/db/file.rs b/src/db/file.rs index b1d694af2..2cf150076 100644 --- a/src/db/file.rs +++ b/src/db/file.rs @@ -6,17 +6,10 @@ use crate::error::Result; use crate::storage::{CompressionAlgorithms, Storage}; -use postgres::Connection; use serde_json::Value; use std::path::{Path, PathBuf}; -pub(crate) use crate::storage::Blob; - -pub(crate) fn get_path(conn: &Connection, path: &str, max_size: usize) -> Result { - Storage::new(conn).get(path, max_size) -} - /// Store all files in a directory and return [[mimetype, filename]] as Json /// /// If there is an S3 Client configured, store files into an S3 bucket; @@ -27,12 +20,11 @@ pub(crate) fn get_path(conn: &Connection, path: &str, max_size: usize) -> Result /// Note that this function is used for uploading both sources /// and files generated by rustdoc. pub fn add_path_into_database>( - conn: &Connection, + storage: &Storage, prefix: &str, path: P, ) -> Result<(Value, CompressionAlgorithms)> { - let mut backend = Storage::new(conn); - let (file_list, algorithms) = backend.store_all(conn, prefix, path.as_ref())?; + let (file_list, algorithms) = storage.store_all(prefix, path.as_ref())?; Ok(( file_list_to_json(file_list.into_iter().collect())?, algorithms, diff --git a/src/db/mod.rs b/src/db/mod.rs index af122fb56..732f1cbdf 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -5,10 +5,8 @@ pub(crate) use self::add_package::add_package_into_database; pub use self::delete::{delete_crate, delete_version}; pub use self::file::add_path_into_database; pub use self::migrate::migrate; -pub use self::pool::{Pool, PoolError}; - -#[cfg(test)] pub(crate) use self::pool::PoolConnection; +pub use self::pool::{Pool, PoolError}; mod add_package; pub mod blacklist; diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index f303c3056..332d9f0d9 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -6,10 +6,10 @@ use crate::db::{add_build_into_database, add_package_into_database, Pool}; use crate::docbuilder::{crates::crates_from_path, Limits}; use crate::error::Result; use crate::storage::CompressionAlgorithms; +use crate::storage::Storage; use crate::utils::{copy_doc_dir, parse_rustc_version, CargoMetadata}; use failure::ResultExt; use log::{debug, info, warn, LevelFilter}; -use postgres::Connection; use rustwide::cmd::{Command, SandboxBuilder}; use rustwide::logging::{self, LogStorage}; use rustwide::toolchain::ToolchainError; @@ -18,6 +18,7 @@ use serde_json::Value; use std::borrow::Cow; use std::collections::HashSet; use std::path::Path; +use std::sync::Arc; const USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)"; const DEFAULT_RUSTWIDE_WORKSPACE: &str = ".rustwide"; @@ -68,12 +69,13 @@ pub struct RustwideBuilder { workspace: Workspace, toolchain: Toolchain, db: Pool, + storage: Arc, rustc_version: String, cpu_limit: Option, } impl RustwideBuilder { - pub fn init(db: Pool) -> Result { + pub fn init(db: Pool, storage: Arc) -> Result { use rustwide::cmd::SandboxImage; let env_workspace_path = ::std::env::var("CRATESFYI_RUSTWIDE_WORKSPACE"); let workspace_path = env_workspace_path @@ -108,6 +110,7 @@ impl RustwideBuilder { workspace, toolchain, db, + storage, rustc_version: String::new(), cpu_limit, }) @@ -245,7 +248,7 @@ impl RustwideBuilder { })?; } - add_path_into_database(&conn, "", &dest)?; + add_path_into_database(&self.storage, "", &dest)?; conn.query( "INSERT INTO config (name, value) VALUES ('rustc_version', $1) \ ON CONFLICT (name) DO UPDATE SET value = $1;", @@ -350,7 +353,7 @@ impl RustwideBuilder { debug!("adding sources into database"); let prefix = format!("sources/{}/{}", name, version); let (files, new_algs) = - add_path_into_database(&conn, &prefix, build.host_source_dir())?; + add_path_into_database(&self.storage, &prefix, build.host_source_dir())?; files_list = Some(files); algs.extend(new_algs); @@ -379,7 +382,7 @@ impl RustwideBuilder { &metadata, )?; } - let new_algs = self.upload_docs(&conn, name, version, local_storage.path())?; + let new_algs = self.upload_docs(name, version, local_storage.path())?; algs.extend(new_algs); }; @@ -576,14 +579,13 @@ impl RustwideBuilder { fn upload_docs( &self, - conn: &Connection, name: &str, version: &str, local_storage: &Path, ) -> Result { debug!("Adding documentation into database"); add_path_into_database( - conn, + &self.storage, &format!("rustdoc/{}/{}", name, version), local_storage, ) diff --git a/src/lib.rs b/src/lib.rs index 7afcca18e..d4276dc9f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ pub use self::config::Config; pub use self::docbuilder::options::DocBuilderOptions; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::RustwideBuilder; +pub use self::storage::Storage; pub use self::web::Server; mod build_queue; diff --git a/src/storage/database.rs b/src/storage/database.rs index 7569e6e15..35d7502f1 100644 --- a/src/storage/database.rs +++ b/src/storage/database.rs @@ -1,19 +1,20 @@ -use super::Blob; +use super::{Blob, StorageTransaction}; +use crate::db::Pool; use chrono::{DateTime, NaiveDateTime, Utc}; use failure::{Error, Fail}; -use postgres::{transaction::Transaction, Connection}; +use postgres::transaction::Transaction; #[derive(Debug, Fail)] #[fail(display = "the path is not present in the database")] struct PathNotFoundError; -pub(crate) struct DatabaseBackend<'a> { - conn: &'a Connection, +pub(crate) struct DatabaseBackend { + pool: Pool, } -impl<'a> DatabaseBackend<'a> { - pub(crate) fn new(conn: &'a Connection) -> Self { - Self { conn } +impl DatabaseBackend { + pub(crate) fn new(pool: Pool) -> Self { + Self { pool } } pub(super) fn get(&self, path: &str, max_size: usize) -> Result { @@ -25,7 +26,7 @@ impl<'a> DatabaseBackend<'a> { // The size limit is checked at the database level, to avoid receiving data altogether if // the limit is exceeded. - let rows = self.conn.query( + let rows = self.pool.get()?.query( "SELECT path, mime, date_updated, compression, (CASE WHEN LENGTH(content) <= $2 THEN content ELSE NULL END) AS content, @@ -62,10 +63,36 @@ impl<'a> DatabaseBackend<'a> { } } - pub(super) fn store_batch(&self, batch: &[Blob], trans: &Transaction) -> Result<(), Error> { + pub(super) fn start_connection(&self) -> Result { + Ok(DatabaseConnection { + conn: self.pool.get()?, + }) + } +} + +pub(super) struct DatabaseConnection { + conn: crate::db::PoolConnection, +} + +impl DatabaseConnection { + pub(super) fn start_storage_transaction( + &self, + ) -> Result, Error> { + Ok(DatabaseStorageTransaction { + transaction: self.conn.transaction()?, + }) + } +} + +pub(super) struct DatabaseStorageTransaction<'a> { + transaction: Transaction<'a>, +} + +impl<'a> StorageTransaction for DatabaseStorageTransaction<'a> { + fn store_batch(&mut self, batch: &[Blob]) -> Result<(), Error> { for blob in batch { let compression = blob.compression.map(|alg| alg as i32); - trans.query( + self.transaction.query( "INSERT INTO files (path, mime, content, compression) VALUES ($1, $2, $3, $4) ON CONFLICT (path) DO UPDATE @@ -75,6 +102,11 @@ impl<'a> DatabaseBackend<'a> { } Ok(()) } + + fn complete(self: Box) -> Result<(), Error> { + self.transaction.commit()?; + Ok(()) + } } #[cfg(test)] @@ -85,8 +117,9 @@ mod tests { #[test] fn test_path_get() { crate::test::wrapper(|env| { - let conn = env.db().conn(); - let backend = DatabaseBackend::new(&conn); + let db = env.db(); + let conn = db.conn(); + let backend = DatabaseBackend::new(db.pool()); let now = Utc::now(); // Add a test file to the database @@ -133,8 +166,8 @@ mod tests { const MAX_SIZE: usize = 1024; crate::test::wrapper(|env| { - let conn = env.db().conn(); - let backend = DatabaseBackend::new(&conn); + let db = env.db(); + let backend = DatabaseBackend::new(db.pool()); let small_blob = Blob { path: "small-blob.bin".into(), @@ -151,14 +184,11 @@ mod tests { compression: None, }; - let transaction = conn.transaction()?; - backend - .store_batch(std::slice::from_ref(&small_blob), &transaction) - .unwrap(); - backend - .store_batch(std::slice::from_ref(&big_blob), &transaction) - .unwrap(); - transaction.commit()?; + let conn = backend.start_connection()?; + let mut transaction = Box::new(conn.start_storage_transaction()?); + transaction.store_batch(std::slice::from_ref(&small_blob))?; + transaction.store_batch(std::slice::from_ref(&big_blob))?; + transaction.complete()?; let blob = backend.get("small-blob.bin", MAX_SIZE).unwrap(); assert_eq!(blob.content.len(), small_blob.content.len()); diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 63e427f4b..a79b589ff 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -3,10 +3,10 @@ pub(crate) mod s3; pub(crate) use self::database::DatabaseBackend; pub(crate) use self::s3::S3Backend; +use crate::db::Pool; use chrono::{DateTime, Utc}; use failure::{err_msg, Error}; use path_slash::PathExt; -use postgres::{transaction::Transaction, Connection}; use std::{ collections::{HashMap, HashSet}, ffi::OsStr, @@ -117,23 +117,29 @@ pub fn get_file_list>(path: P) -> Result, Error> { Ok(files) } -pub(crate) enum Storage<'a> { - Database(DatabaseBackend<'a>), - S3(S3Backend<'a>), +enum StorageBackend { + Database(DatabaseBackend), + S3(S3Backend), } -impl<'a> Storage<'a> { - pub(crate) fn new(conn: &'a Connection) -> Self { - if let Some(c) = s3::s3_client() { - Storage::from(S3Backend::new(c, s3::S3_BUCKET_NAME)) +pub struct Storage { + backend: StorageBackend, +} + +impl Storage { + pub fn new(pool: Pool) -> Self { + let backend = if let Some(c) = s3::s3_client() { + StorageBackend::S3(S3Backend::new(c, s3::S3_BUCKET_NAME)) } else { - DatabaseBackend::new(conn).into() - } + StorageBackend::Database(DatabaseBackend::new(pool)) + }; + Storage { backend } } + pub(crate) fn get(&self, path: &str, max_size: usize) -> Result { - let mut blob = match self { - Self::Database(db) => db.get(path, max_size), - Self::S3(s3) => s3.get(path, max_size), + let mut blob = match &self.backend { + StorageBackend::Database(db) => db.get(path, max_size), + StorageBackend::S3(s3) => s3.get(path, max_size), }?; if let Some(alg) = blob.compression { blob.content = decompress(blob.content.as_slice(), alg, max_size)?; @@ -142,13 +148,6 @@ impl<'a> Storage<'a> { Ok(blob) } - fn store_batch(&mut self, batch: &[Blob], trans: &Transaction) -> Result<(), Error> { - match self { - Self::Database(db) => db.store_batch(batch, trans), - Self::S3(s3) => s3.store_batch(batch), - } - } - // Store all files in `root_dir` into the backend under `prefix`. // // If the environment is configured with S3 credentials, this will upload to S3; @@ -156,12 +155,19 @@ impl<'a> Storage<'a> { // // This returns (map, set). pub(crate) fn store_all( - &mut self, - conn: &Connection, + &self, prefix: &str, root_dir: &Path, ) -> Result<(HashMap, HashSet), Error> { - let trans = conn.transaction()?; + let conn; + let mut trans: Box = match &self.backend { + StorageBackend::Database(db) => { + conn = db.start_connection()?; + Box::new(conn.start_storage_transaction()?) + } + StorageBackend::S3(s3) => Box::new(s3.start_storage_transaction()?), + }; + let mut file_paths_and_mimes = HashMap::new(); let mut algs = HashSet::with_capacity(1); @@ -201,14 +207,28 @@ impl<'a> Storage<'a> { if batch.is_empty() { break; } - self.store_batch(&batch, &trans)?; + trans.store_batch(&batch)?; } - trans.commit()?; + trans.complete()?; Ok((file_paths_and_mimes, algs)) } } +impl std::fmt::Debug for Storage { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self.backend { + StorageBackend::Database(_) => write!(f, "database-backed storage"), + StorageBackend::S3(_) => write!(f, "S3-backed storage"), + } + } +} + +trait StorageTransaction { + fn store_batch(&mut self, batch: &[Blob]) -> Result<(), Error>; + fn complete(self: Box) -> Result<(), Error>; +} + // public for benchmarking pub fn compress(content: impl Read, algorithm: CompressionAlgorithm) -> Result, Error> { match algorithm { @@ -254,18 +274,6 @@ fn detect_mime(file_path: &Path) -> Result<&'static str, Error> { }) } -impl<'a> From> for Storage<'a> { - fn from(db: DatabaseBackend<'a>) -> Self { - Self::Database(db) - } -} - -impl<'a> From> for Storage<'a> { - fn from(db: S3Backend<'a>) -> Self { - Self::S3(db) - } -} - #[cfg(test)] mod test { use super::*; @@ -293,9 +301,10 @@ mod test { } wrapper(|env| { let db = env.db(); - let conn = db.conn(); - let mut backend = Storage::Database(DatabaseBackend::new(&conn)); - let (stored_files, _algs) = backend.store_all(&conn, "", dir.path()).unwrap(); + let backend = Storage { + backend: StorageBackend::Database(DatabaseBackend::new(db.pool())), + }; + let (stored_files, _algs) = backend.store_all("", dir.path()).unwrap(); assert_eq!(stored_files.len(), blobs.len()); for blob in blobs { let name = Path::new(&blob.path); @@ -326,9 +335,10 @@ mod test { } wrapper(|env| { let db = env.db(); - let conn = db.conn(); - let mut backend = Storage::Database(DatabaseBackend::new(&conn)); - let (stored_files, _algs) = backend.store_all(&conn, "rustdoc", dir.path()).unwrap(); + let backend = Storage { + backend: StorageBackend::Database(DatabaseBackend::new(db.pool())), + }; + let (stored_files, _algs) = backend.store_all("rustdoc", dir.path()).unwrap(); assert_eq!(stored_files.len(), files.len()); for name in &files { let name = Path::new(name); diff --git a/src/storage/s3.rs b/src/storage/s3.rs index 54d550a56..bfd6d7b3a 100644 --- a/src/storage/s3.rs +++ b/src/storage/s3.rs @@ -1,6 +1,7 @@ -use super::Blob; +use super::{Blob, StorageTransaction}; use chrono::{DateTime, NaiveDateTime, Utc}; use failure::Error; +use futures::stream::{FuturesUnordered, Stream}; use futures::Future; use log::{error, warn}; use rusoto_core::region::Region; @@ -16,18 +17,16 @@ pub(crate) use test::TestS3; pub(crate) static S3_BUCKET_NAME: &str = "rust-docs-rs"; -pub(crate) struct S3Backend<'a> { +pub(crate) struct S3Backend { client: S3Client, - bucket: &'a str, - runtime: Runtime, + bucket: String, } -impl<'a> S3Backend<'a> { - pub(crate) fn new(client: S3Client, bucket: &'a str) -> Self { +impl S3Backend { + pub(crate) fn new(client: S3Client, bucket: &str) -> Self { Self { client, - bucket, - runtime: Runtime::new().unwrap(), + bucket: bucket.into(), } } @@ -63,19 +62,31 @@ impl<'a> S3Backend<'a> { }) } - pub(super) fn store_batch(&mut self, batch: &[Blob]) -> Result<(), Error> { - use futures::stream::FuturesUnordered; - use futures::stream::Stream; + pub(super) fn start_storage_transaction(&self) -> Result { + Ok(S3StorageTransaction { + s3: self, + runtime: Runtime::new()?, + }) + } +} + +pub(super) struct S3StorageTransaction<'a> { + s3: &'a S3Backend, + runtime: Runtime, +} +impl<'a> StorageTransaction for S3StorageTransaction<'a> { + fn store_batch(&mut self, batch: &[Blob]) -> Result<(), Error> { let mut attempts = 0; loop { let mut futures = FuturesUnordered::new(); for blob in batch { futures.push( - self.client + self.s3 + .client .put_object(PutObjectRequest { - bucket: self.bucket.to_string(), + bucket: self.s3.bucket.to_string(), key: blob.path.clone(), body: Some(blob.content.clone().into()), content_type: Some(blob.mime.clone()), @@ -103,6 +114,10 @@ impl<'a> S3Backend<'a> { } Ok(()) } + + fn complete(self: Box) -> Result<(), Error> { + Ok(()) + } } fn parse_timespec(mut raw: &str) -> Result, Error> { diff --git a/src/storage/s3/test.rs b/src/storage/s3/test.rs index be915379f..55a0828ee 100644 --- a/src/storage/s3/test.rs +++ b/src/storage/s3/test.rs @@ -5,7 +5,7 @@ use rusoto_s3::{ }; use std::cell::RefCell; -pub(crate) struct TestS3(RefCell>); +pub(crate) struct TestS3(RefCell); impl TestS3 { pub(crate) fn new() -> Self { @@ -24,7 +24,11 @@ impl TestS3 { TestS3(RefCell::new(S3Backend::new(client, bucket))) } pub(crate) fn upload(&self, blobs: &[Blob]) -> Result<(), Error> { - self.0.borrow_mut().store_batch(blobs) + let s3 = self.0.borrow(); + let mut transaction = Box::new(s3.start_storage_transaction()?); + transaction.store_batch(blobs)?; + transaction.complete()?; + Ok(()) } pub(crate) fn assert_404(&self, path: &'static str) { use rusoto_core::RusotoError; @@ -45,7 +49,7 @@ impl TestS3 { assert_blob_eq(blob, &actual); } - pub(crate) fn with_client(&self, f: impl FnOnce(&mut S3Backend<'static>)) { + pub(crate) fn with_client(&self, f: impl FnOnce(&mut S3Backend)) { f(&mut self.0.borrow_mut()) } } diff --git a/src/test/fakes.rs b/src/test/fakes.rs index 4b162a2b7..4d30760ed 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -1,13 +1,16 @@ use super::TestDatabase; use crate::docbuilder::BuildResult; use crate::index::api::RegistryCrateData; +use crate::storage::Storage; use crate::utils::{Dependency, MetadataPackage, Target}; use chrono::{DateTime, Utc}; use failure::Error; +use std::sync::Arc; #[must_use = "FakeRelease does nothing until you call .create()"] pub(crate) struct FakeRelease<'a> { db: &'a TestDatabase, + storage: Arc, package: MetadataPackage, build_result: BuildResult, /// name, content @@ -24,9 +27,10 @@ pub(crate) struct FakeRelease<'a> { } impl<'a> FakeRelease<'a> { - pub(super) fn new(db: &'a TestDatabase) -> Self { + pub(super) fn new(db: &'a TestDatabase, storage: Arc) -> Self { FakeRelease { db, + storage, package: MetadataPackage { id: "fake-package-id".into(), name: "fake-package".into(), @@ -175,6 +179,7 @@ impl<'a> FakeRelease<'a> { let mut source_meta = None; let mut algs = HashSet::new(); if self.build_result.successful { + let storage = self.storage.clone(); let upload_files = |prefix: &str, files: &[(&str, &[u8])], target: Option<&str>| { let mut path_prefix = tempdir.path().join(prefix); if let Some(target) = target { @@ -200,7 +205,7 @@ impl<'a> FakeRelease<'a> { target.unwrap_or("") ); log::debug!("adding directory {} from {}", prefix, path_prefix.display()); - crate::db::add_path_into_database(&db.conn(), &prefix, path_prefix) + crate::db::add_path_into_database(&storage, &prefix, path_prefix) }; let index = [&package.name, "index.html"].join("/"); diff --git a/src/test/mod.rs b/src/test/mod.rs index 95239ff58..b9cf88ddb 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -2,6 +2,7 @@ mod fakes; use crate::db::{Pool, PoolConnection}; use crate::storage::s3::TestS3; +use crate::storage::Storage; use crate::web::Server; use crate::BuildQueue; use crate::Config; @@ -96,6 +97,7 @@ pub(crate) struct TestEnvironment { build_queue: OnceCell>, config: OnceCell>, db: OnceCell, + storage: OnceCell>, frontend: OnceCell, s3: OnceCell, } @@ -112,6 +114,7 @@ impl TestEnvironment { build_queue: OnceCell::new(), config: OnceCell::new(), db: OnceCell::new(), + storage: OnceCell::new(), frontend: OnceCell::new(), s3: OnceCell::new(), } @@ -154,19 +157,30 @@ impl TestEnvironment { .clone() } + pub(crate) fn storage(&self) -> Arc { + self.storage + .get_or_init(|| Arc::new(Storage::new(self.db().pool()))) + .clone() + } + pub(crate) fn db(&self) -> &TestDatabase { self.db .get_or_init(|| TestDatabase::new(&self.config()).expect("failed to initialize the db")) } pub(crate) fn frontend(&self) -> &TestFrontend { - self.frontend - .get_or_init(|| TestFrontend::new(self.db(), self.config(), self.build_queue())) + self.frontend.get_or_init(|| { + TestFrontend::new(self.db(), self.config(), self.build_queue(), self.storage()) + }) } pub(crate) fn s3(&self) -> &TestS3 { self.s3.get_or_init(TestS3::new) } + + pub(crate) fn fake_release(&self) -> fakes::FakeRelease { + fakes::FakeRelease::new(self.db(), self.storage()) + } } pub(crate) struct TestDatabase { @@ -205,10 +219,6 @@ impl TestDatabase { .get() .expect("failed to get a connection out of the pool") } - - pub(crate) fn fake_release(&self) -> fakes::FakeRelease { - fakes::FakeRelease::new(self) - } } impl Drop for TestDatabase { @@ -229,7 +239,12 @@ pub(crate) struct TestFrontend { } impl TestFrontend { - fn new(db: &TestDatabase, config: Arc, build_queue: Arc) -> Self { + fn new( + db: &TestDatabase, + config: Arc, + build_queue: Arc, + storage: Arc, + ) -> Self { Self { server: Server::start( Some("127.0.0.1:0"), @@ -237,6 +252,7 @@ impl TestFrontend { db.pool.clone(), config, build_queue, + storage, ) .expect("failed to start the web server"), client: Client::new(), diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 50006aefc..1e146c591 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -4,6 +4,7 @@ use crate::{ db::Pool, + storage::Storage, utils::{queue_builder, update_release_activity, GithubUpdater}, BuildQueue, Config, DocBuilder, DocBuilderOptions, }; @@ -46,6 +47,7 @@ pub fn start_daemon( config: Arc, db: Pool, build_queue: Arc, + storage: Arc, enable_registry_watcher: bool, ) -> Result<(), Error> { const CRATE_VARIABLES: &[&str] = &["CRATESFYI_PREFIX"]; @@ -70,12 +72,13 @@ pub fn start_daemon( // build new crates every minute let cloned_db = db.clone(); let cloned_build_queue = build_queue.clone(); + let cloned_storage = storage.clone(); thread::Builder::new() .name("build queue reader".to_string()) .spawn(move || { let doc_builder = DocBuilder::new(opts(), cloned_db.clone(), cloned_build_queue.clone()); - queue_builder(doc_builder, cloned_db, cloned_build_queue).unwrap(); + queue_builder(doc_builder, cloned_db, cloned_build_queue, cloned_storage).unwrap(); }) .unwrap(); @@ -110,7 +113,7 @@ pub fn start_daemon( // at least start web server info!("Starting web server"); - crate::Server::start(None, false, db, config, build_queue)?; + crate::Server::start(None, false, db, config, build_queue, storage)?; Ok(()) } diff --git a/src/utils/queue_builder.rs b/src/utils/queue_builder.rs index 6e9b66e9c..1eaef5478 100644 --- a/src/utils/queue_builder.rs +++ b/src/utils/queue_builder.rs @@ -1,4 +1,6 @@ -use crate::{db::Pool, docbuilder::RustwideBuilder, utils::pubsubhubbub, BuildQueue, DocBuilder}; +use crate::{ + db::Pool, docbuilder::RustwideBuilder, utils::pubsubhubbub, BuildQueue, DocBuilder, Storage, +}; use failure::Error; use log::{debug, error, info, warn}; use std::panic::{catch_unwind, AssertUnwindSafe}; @@ -11,6 +13,7 @@ pub fn queue_builder( mut doc_builder: DocBuilder, db: Pool, build_queue: Arc, + storage: Arc, ) -> Result<(), Error> { /// Represents the current state of the builder thread. enum BuilderState { @@ -25,7 +28,7 @@ pub fn queue_builder( QueueInProgress(usize), } - let mut builder = RustwideBuilder::init(db)?; + let mut builder = RustwideBuilder::init(db, storage)?; let mut status = BuilderState::Fresh; diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs index cb27dd0f5..fc0adad1b 100644 --- a/src/web/crate_details.rs +++ b/src/web/crate_details.rs @@ -340,19 +340,19 @@ mod tests { wrapper(|env| { let db = env.db(); - db.fake_release().name("foo").version("0.0.1").create()?; - db.fake_release().name("foo").version("0.0.2").create()?; - db.fake_release() + env.fake_release().name("foo").version("0.0.1").create()?; + env.fake_release().name("foo").version("0.0.2").create()?; + env.fake_release() .name("foo") .version("0.0.3") .build_result_successful(false) .create()?; - db.fake_release() + env.fake_release() .name("foo") .version("0.0.4") .yanked(true) .create()?; - db.fake_release() + env.fake_release() .name("foo") .version("0.0.5") .build_result_successful(false) @@ -373,17 +373,17 @@ mod tests { wrapper(|env| { let db = env.db(); - db.fake_release() + env.fake_release() .name("foo") .version("0.0.1") .build_result_successful(false) .create()?; - db.fake_release() + env.fake_release() .name("foo") .version("0.0.2") .build_result_successful(false) .create()?; - db.fake_release() + env.fake_release() .name("foo") .version("0.0.3") .yanked(true) @@ -401,18 +401,18 @@ mod tests { wrapper(|env| { let db = env.db(); - db.fake_release().name("foo").version("0.0.1").create()?; - db.fake_release() + env.fake_release().name("foo").version("0.0.1").create()?; + env.fake_release() .name("foo") .version("0.0.2") .build_result_successful(false) .create()?; - db.fake_release() + env.fake_release() .name("foo") .version("0.0.3") .yanked(true) .create()?; - db.fake_release().name("foo").version("0.0.4").create()?; + env.fake_release().name("foo").version("0.0.4").create()?; assert_last_successful_build_equals(&db, "foo", "0.0.1", None)?; assert_last_successful_build_equals(&db, "foo", "0.0.2", Some("0.0.4"))?; @@ -428,25 +428,25 @@ mod tests { let db = env.db(); // Add new releases of 'foo' out-of-order since CrateDetails should sort them descending - db.fake_release().name("foo").version("0.1.0").create()?; - db.fake_release().name("foo").version("0.1.1").create()?; - db.fake_release() + env.fake_release().name("foo").version("0.1.0").create()?; + env.fake_release().name("foo").version("0.1.1").create()?; + env.fake_release() .name("foo") .version("0.3.0") .build_result_successful(false) .create()?; - db.fake_release().name("foo").version("1.0.0").create()?; - db.fake_release().name("foo").version("0.12.0").create()?; - db.fake_release() + env.fake_release().name("foo").version("1.0.0").create()?; + env.fake_release().name("foo").version("0.12.0").create()?; + env.fake_release() .name("foo") .version("0.2.0") .yanked(true) .create()?; - db.fake_release() + env.fake_release() .name("foo") .version("0.2.0-alpha") .create()?; - db.fake_release() + env.fake_release() .name("foo") .version("0.0.1") .build_result_successful(false) @@ -517,9 +517,9 @@ mod tests { wrapper(|env| { let db = env.db(); - db.fake_release().name("foo").version("0.0.1").create()?; - db.fake_release().name("foo").version("0.0.3").create()?; - db.fake_release().name("foo").version("0.0.2").create()?; + env.fake_release().name("foo").version("0.0.1").create()?; + env.fake_release().name("foo").version("0.0.3").create()?; + env.fake_release().name("foo").version("0.0.2").create()?; for version in &["0.0.1", "0.0.2", "0.0.3"] { let details = CrateDetails::new(&db.conn(), "foo", version).unwrap(); @@ -535,13 +535,13 @@ mod tests { wrapper(|env| { let db = env.db(); - db.fake_release().name("foo").version("0.0.1").create()?; - db.fake_release() + env.fake_release().name("foo").version("0.0.1").create()?; + env.fake_release() .name("foo") .version("0.0.3") .yanked(true) .create()?; - db.fake_release().name("foo").version("0.0.2").create()?; + env.fake_release().name("foo").version("0.0.2").create()?; for version in &["0.0.1", "0.0.2", "0.0.3"] { let details = CrateDetails::new(&db.conn(), "foo", version).unwrap(); @@ -557,17 +557,17 @@ mod tests { wrapper(|env| { let db = env.db(); - db.fake_release() + env.fake_release() .name("foo") .version("0.0.1") .yanked(true) .create()?; - db.fake_release() + env.fake_release() .name("foo") .version("0.0.3") .yanked(true) .create()?; - db.fake_release() + env.fake_release() .name("foo") .version("0.0.2") .yanked(true) @@ -585,9 +585,7 @@ mod tests { #[test] fn releases_dropdowns_is_correct() { wrapper(|env| { - let db = env.db(); - - db.fake_release() + env.fake_release() .name("binary") .version("0.1.0") .binary(true) diff --git a/src/web/extensions.rs b/src/web/extensions.rs index b0b78074f..a8957f650 100644 --- a/src/web/extensions.rs +++ b/src/web/extensions.rs @@ -1,5 +1,6 @@ use crate::config::Config; use crate::db::Pool; +use crate::storage::Storage; use crate::web::page::TemplateData; use crate::BuildQueue; use iron::{BeforeMiddleware, IronResult, Request}; @@ -10,6 +11,7 @@ pub(super) struct InjectExtensions { pub(super) build_queue: Arc, pub(super) pool: Pool, pub(super) config: Arc, + pub(super) storage: Arc, pub(super) template_data: Arc, } @@ -19,6 +21,7 @@ impl BeforeMiddleware for InjectExtensions { .insert::(self.build_queue.clone()); req.extensions.insert::(self.pool.clone()); req.extensions.insert::(self.config.clone()); + req.extensions.insert::(self.storage.clone()); req.extensions .insert::(self.template_data.clone()); @@ -37,4 +40,5 @@ macro_rules! key { key!(BuildQueue => Arc); key!(Pool => Pool); key!(Config => Arc); +key!(Storage => Arc); key!(TemplateData => Arc); diff --git a/src/web/file.rs b/src/web/file.rs index c94ce36d0..985ab5679 100644 --- a/src/web/file.rs +++ b/src/web/file.rs @@ -1,23 +1,22 @@ //! Database based file handler -use crate::db::Pool; -use crate::{db, error::Result, Config}; +use crate::storage::{Blob, Storage}; +use crate::{error::Result, Config}; use iron::{status, Handler, IronError, IronResult, Request, Response}; -use postgres::Connection; #[derive(Debug)] -pub(crate) struct File(pub(crate) db::file::Blob); +pub(crate) struct File(pub(crate) Blob); impl File { /// Gets file from database - pub fn from_path(conn: &Connection, path: &str, config: &Config) -> Result { + pub fn from_path(storage: &Storage, path: &str, config: &Config) -> Result { let max_size = if path.ends_with(".html") { config.max_file_size_html } else { config.max_file_size }; - Ok(File(db::file::get_path(conn, path, max_size)?)) + Ok(File(storage.get(path, max_size)?)) } /// Consumes File and creates a iron response @@ -58,9 +57,9 @@ pub struct DatabaseFileHandler; impl Handler for DatabaseFileHandler { fn handle(&self, req: &mut Request) -> IronResult { let path = req.url.path().join("/"); - let conn = extension!(req, Pool).get()?; + let storage = extension!(req, Storage); let config = extension!(req, Config); - if let Ok(file) = File::from_path(&conn, &path, &config) { + if let Ok(file) = File::from_path(&storage, &path, &config) { Ok(file.serve()) } else { Err(IronError::new( @@ -80,13 +79,12 @@ mod tests { #[test] fn file_roundtrip() { wrapper(|env| { - let db = env.db(); let now = Utc::now(); - db.fake_release().create()?; + env.fake_release().create()?; let mut file = File::from_path( - &*db.conn(), + &env.storage(), "rustdoc/fake-package/1.0.0/fake-package/index.html", &env.config(), ) @@ -114,9 +112,7 @@ mod tests { config.max_file_size_html = MAX_HTML_SIZE; }); - let db = env.db(); - - db.fake_release() + env.fake_release() .name("dummy") .version("0.1.0") .rustdoc_file("small.html", &[b'A'; MAX_HTML_SIZE / 2] as &[u8]) @@ -129,7 +125,7 @@ mod tests { let file = |path| { File::from_path( - &db.conn(), + &env.storage(), &format!("rustdoc/dummy/0.1.0/{}", path), &env.config(), ) diff --git a/src/web/metrics.rs b/src/web/metrics.rs index e0eeeaf37..c2e156374 100644 --- a/src/web/metrics.rs +++ b/src/web/metrics.rs @@ -347,14 +347,12 @@ mod tests { #[test] fn releases() { wrapper(|env| { - env.db() - .fake_release() + env.fake_release() .name("rcc") .version("0.0.0") .repo("https://github.com/jyn514/rcc") .create()?; - env.db() - .fake_release() + env.fake_release() .name("rcc") .version("1.0.0") .build_result_successful(false) @@ -391,13 +389,8 @@ mod tests { #[test] fn crates() { wrapper(|env| { - env.db() - .fake_release() - .name("rcc") - .version("0.0.0") - .create()?; - env.db() - .fake_release() + env.fake_release().name("rcc").version("0.0.0").create()?; + env.fake_release() .name("hexponent") .version("0.2.0") .create()?; diff --git a/src/web/mod.rs b/src/web/mod.rs index d74d5a92c..53bca3942 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -77,7 +77,7 @@ mod rustdoc; mod sitemap; mod source; -use crate::{config::Config, db::Pool, impl_webpage, BuildQueue}; +use crate::{config::Config, db::Pool, impl_webpage, BuildQueue, Storage}; use chrono::{DateTime, Utc}; use extensions::InjectExtensions; use failure::Error; @@ -127,11 +127,13 @@ impl CratesfyiHandler { config: Arc, template_data: Arc, build_queue: Arc, + storage: Arc, ) -> CratesfyiHandler { let inject_extensions = InjectExtensions { build_queue, pool, config, + storage, template_data, }; @@ -394,6 +396,7 @@ impl Server { db: Pool, config: Arc, build_queue: Arc, + storage: Arc, ) -> Result { // Initialize templates let template_data = Arc::new(TemplateData::new(&*db.get()?)?); @@ -407,6 +410,7 @@ impl Server { config, template_data, build_queue, + storage, ); info!("Running docs.rs web server on http://{}", server.addr()); Ok(server) @@ -418,6 +422,7 @@ impl Server { config: Arc, template_data: Arc, build_queue: Arc, + storage: Arc, ) -> Self { // poke all the metrics counters to instantiate and register them metrics::TOTAL_BUILDS.inc_by(0); @@ -427,7 +432,7 @@ impl Server { metrics::UPLOADED_FILES_TOTAL.inc_by(0); metrics::FAILED_DB_CONNECTIONS.inc_by(0); - let cratesfyi = CratesfyiHandler::new(pool, config, template_data, build_queue); + let cratesfyi = CratesfyiHandler::new(pool, config, template_data, build_queue, storage); let inner = Iron::new(cratesfyi) .http(addr) .unwrap_or_else(|_| panic!("Failed to bind to socket on {}", addr)); @@ -635,8 +640,8 @@ mod test { use kuchiki::traits::TendrilSink; use serde_json::json; - fn release(version: &str, db: &TestDatabase) -> i32 { - db.fake_release() + fn release(version: &str, env: &TestEnvironment) -> i32 { + env.fake_release() .name("foo") .version(version) .create() @@ -674,8 +679,7 @@ mod test { #[test] fn test_show_clipboard_for_crate_pages() { wrapper(|env| { - env.db() - .fake_release() + env.fake_release() .name("fake_crate") .version("0.0.1") .source_file("test.rs", &[]) @@ -697,8 +701,7 @@ mod test { #[test] fn test_hide_clipboard_for_non_crate_pages() { wrapper(|env| { - env.db() - .fake_release() + env.fake_release() .name("fake_crate") .version("0.0.1") .create() @@ -734,8 +737,7 @@ mod test { #[test] fn binary_docs_redirect_to_crate() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("bat") .version("0.2.0") .binary(true) @@ -755,8 +757,7 @@ mod test { #[test] fn can_view_source() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("regex") .version("0.3.0") .source_file("src/main.rs", br#"println!("definitely valid rust")"#) @@ -778,7 +779,7 @@ mod test { wrapper(|env| { let db = env.db(); let version = |v| version(v, db); - let release = |v| release(v, db); + let release = |v| release(v, env); release("0.3.1-pre"); assert_eq!(version(Some("*")), semver("0.3.1-pre")); @@ -804,14 +805,14 @@ mod test { wrapper(|env| { let db = env.db(); - let release_id = release("0.3.0", db); + let release_id = release("0.3.0", env); let query = "UPDATE releases SET yanked = true WHERE id = $1 AND version = '0.3.0'"; db.conn().query(query, &[&release_id]).unwrap(); assert_eq!(version(None, db), None); assert_eq!(version(Some("0.3"), db), None); - release("0.1.0+4.1", db); + release("0.1.0+4.1", env); assert_eq!(version(Some("0.1.0+4.1"), db), exact("0.1.0+4.1")); assert_eq!(version(None, db), semver("0.1.0+4.1")); @@ -825,10 +826,10 @@ mod test { wrapper(|env| { let db = env.db(); - release("0.1.0+4.1", db); - release("0.1.1", db); + release("0.1.0+4.1", env); + release("0.1.1", env); assert_eq!(version(None, db), semver("0.1.1")); - release("0.5.1+zstd.1.4.4", db); + release("0.5.1+zstd.1.4.4", env); assert_eq!(version(None, db), semver("0.5.1+zstd.1.4.4")); assert_eq!(version(Some("0.5"), db), semver("0.5.1+zstd.1.4.4")); assert_eq!( diff --git a/src/web/releases.rs b/src/web/releases.rs index b893b503f..34d6ff080 100644 --- a/src/web/releases.rs +++ b/src/web/releases.rs @@ -690,26 +690,26 @@ mod tests { wrapper(|env| { let db = env.db(); - db.fake_release().name("foo").version("0.0.0").create()?; - db.fake_release() + env.fake_release().name("foo").version("0.0.0").create()?; + env.fake_release() .name("bar-foo") .version("0.0.0") .create()?; - db.fake_release() + env.fake_release() .name("foo-bar") .version("0.0.1") .create()?; - db.fake_release().name("fo0").version("0.0.0").create()?; - db.fake_release() + env.fake_release().name("fo0").version("0.0.0").create()?; + env.fake_release() .name("fool") .version("0.0.0") .build_result_successful(false) .create()?; - db.fake_release() + env.fake_release() .name("freakin") .version("0.0.0") .create()?; - db.fake_release() + env.fake_release() .name("something unreleated") .version("0.0.0") .create()?; @@ -736,7 +736,7 @@ mod tests { let releases = ["regex", "regex-", "regex-syntax"]; for release in releases.iter() { - db.fake_release().name(release).version("0.0.0").create()?; + env.fake_release().name(release).version("0.0.0").create()?; } let near_matches = ["Regex", "rEgex", "reGex", "regEx", "regeX"]; @@ -760,7 +760,7 @@ mod tests { fn unsuccessful_not_shown() { wrapper(|env| { let db = env.db(); - db.fake_release() + env.fake_release() .name("regex") .version("0.0.0") .build_result_successful(false) @@ -780,7 +780,7 @@ mod tests { fn yanked_not_shown() { wrapper(|env| { let db = env.db(); - db.fake_release() + env.fake_release() .name("regex") .version("0.0.0") .yanked(true) @@ -800,7 +800,7 @@ mod tests { fn fuzzily_match() { wrapper(|env| { let db = env.db(); - db.fake_release().name("regex").version("0.0.0").create()?; + env.fake_release().name("regex").version("0.0.0").create()?; let (num_results, results) = get_search_results(&db.conn(), "redex", 1, 100); assert_eq!(num_results, 1); @@ -818,7 +818,7 @@ mod tests { // fn search_descriptions() { // wrapper(|env| { // let db = env.db(); - // db.fake_release() + // env.fake_release() // .name("something_completely_unrelated") // .description("Supercalifragilisticexpialidocious") // .create()?; @@ -843,10 +843,10 @@ mod tests { wrapper(|env| { let db = env.db(); - db.fake_release().name("something_magical").create()?; - db.fake_release().name("something_sinister").create()?; - db.fake_release().name("something_fantastical").create()?; - db.fake_release() + env.fake_release().name("something_magical").create()?; + env.fake_release().name("something_sinister").create()?; + env.fake_release().name("something_fantastical").create()?; + env.fake_release() .name("something_completely_unrelated") .create()?; @@ -866,10 +866,10 @@ mod tests { fn search_offsets() { wrapper(|env| { let db = env.db(); - db.fake_release().name("something_magical").create()?; - db.fake_release().name("something_sinister").create()?; - db.fake_release().name("something_fantastical").create()?; - db.fake_release() + env.fake_release().name("something_magical").create()?; + env.fake_release().name("something_sinister").create()?; + env.fake_release().name("something_fantastical").create()?; + env.fake_release() .name("something_completely_unrelated") .create()?; @@ -892,25 +892,25 @@ mod tests { fn release_dates() { wrapper(|env| { let db = env.db(); - db.fake_release() + env.fake_release() .name("somethang") .release_time(Utc.ymd(2021, 4, 16).and_hms(4, 33, 50)) .version("0.3.0") .description("this is the correct choice") .create()?; - db.fake_release() + env.fake_release() .name("somethang") .release_time(Utc.ymd(2020, 4, 16).and_hms(4, 33, 50)) .description("second") .version("0.2.0") .create()?; - db.fake_release() + env.fake_release() .name("somethang") .release_time(Utc.ymd(2019, 4, 16).and_hms(4, 33, 50)) .description("third") .version("0.1.0") .create()?; - db.fake_release() + env.fake_release() .name("somethang") .release_time(Utc.ymd(2018, 4, 16).and_hms(4, 33, 50)) .description("fourth") @@ -936,15 +936,15 @@ mod tests { // fn fuzzy_over_description() { // wrapper(|env| { // let db = env.db(); - // db.fake_release() + // env.fake_release() // .name("name_better_than_description") // .description("this is the correct choice") // .create()?; - // db.fake_release() + // env.fake_release() // .name("im_completely_unrelated") // .description("name_better_than_description") // .create()?; - // db.fake_release() + // env.fake_release() // .name("i_have_zero_relation_whatsoever") // .create()?; // @@ -975,10 +975,10 @@ mod tests { fn dont_return_unrelated() { wrapper(|env| { let db = env.db(); - db.fake_release().name("match").create()?; - db.fake_release().name("matcher").create()?; - db.fake_release().name("matchest").create()?; - db.fake_release() + env.fake_release().name("match").create()?; + env.fake_release().name("matcher").create()?; + env.fake_release().name("matchest").create()?; + env.fake_release() .name("i_am_useless_and_mean_nothing") .create()?; @@ -999,9 +999,9 @@ mod tests { fn order_by_downloads() { wrapper(|env| { let db = env.db(); - db.fake_release().name("matca").downloads(100).create()?; - db.fake_release().name("matcb").downloads(10).create()?; - db.fake_release().name("matcc").downloads(1).create()?; + env.fake_release().name("matca").downloads(100).create()?; + env.fake_release().name("matcb").downloads(10).create()?; + env.fake_release().name("matcc").downloads(1).create()?; let (num_results, results) = get_search_results(&db.conn(), "match", 1, 100); assert_eq!(num_results, 3); @@ -1017,12 +1017,11 @@ mod tests { } fn releases_link_test(path: &str, env: &TestEnvironment) -> Result<(), Error> { - let db = env.db(); - db.fake_release() + env.fake_release() .name("crate_that_succeeded") .version("0.1.0") .create()?; - db.fake_release() + env.fake_release() .name("crate_that_failed") .version("0.1.0") .build_result_successful(false) @@ -1059,7 +1058,7 @@ mod tests { fn search() { wrapper(|env| { let web = env.frontend(); - env.db().fake_release().name("some_random_crate").create()?; + env.fake_release().name("some_random_crate").create()?; assert_success("/releases/search?query=some_random_crate", web) }) } @@ -1097,9 +1096,8 @@ mod tests { let web = env.frontend(); assert_success("/releases/feed", web)?; - env.db().fake_release().name("some_random_crate").create()?; - env.db() - .fake_release() + env.fake_release().name("some_random_crate").create()?; + env.fake_release() .name("some_random_crate_that_failed") .build_result_successful(false) .create()?; @@ -1155,8 +1153,7 @@ mod tests { fn authors_page() { wrapper(|env| { let web = env.frontend(); - env.db() - .fake_release() + env.fake_release() .name("some_random_crate") .author("frankenstein ") .create()?; diff --git a/src/web/rustdoc.rs b/src/web/rustdoc.rs index 322c0f972..cab7513d4 100644 --- a/src/web/rustdoc.rs +++ b/src/web/rustdoc.rs @@ -7,14 +7,13 @@ use crate::{ crate_details::CrateDetails, error::Nope, file::File, match_version, metrics, page::WebPage, redirect_base, MatchSemver, }, - Config, + Config, Storage, }; use iron::{ headers::{CacheControl, CacheDirective, Expires, HttpDate}, modifiers::Redirect, status, Handler, IronError, IronResult, Plugin, Request, Response, Url, }; -use postgres::Connection; use router::Router; use serde::Serialize; @@ -105,12 +104,12 @@ pub fn rustdoc_redirector_handler(req: &mut Request) -> IronResult { // this URL is actually from a crate-internal path, serve it there instead return rustdoc_html_server_handler(req); } else { - let conn = extension!(req, Pool).get()?; + let storage = extension!(req, Storage); let config = extension!(req, Config); let path = req.url.path(); let path = path.join("/"); - match File::from_path(&conn, &path, &config) { + match File::from_path(&storage, &path, &config) { Ok(f) => return Ok(f.serve()), Err(..) => return Err(IronError::new(Nope::ResourceNotFound, status::NotFound)), } @@ -216,8 +215,10 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { router.find("version"), ); - let conn = extension!(req, Pool).get()?; + let pool = extension!(req, Pool); + let conn = pool.get()?; let config = extension!(req, Config); + let storage = extension!(req, Storage); let mut req_path = req.url.path(); // Remove the name and version from the path @@ -297,14 +298,14 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { } // Attempt to load the file from the database - let file = if let Ok(file) = File::from_path(&conn, &path, &config) { + let file = if let Ok(file) = File::from_path(&storage, &path, &config) { file } else { // If it fails, we try again with /index.html at the end path.push_str("/index.html"); req_path.push("index.html"); - File::from_path(&conn, &path, &config) + File::from_path(&storage, &path, &config) .map_err(|_| IronError::new(Nope::ResourceNotFound, status::NotFound))? }; @@ -351,7 +352,7 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { "/{}/{}/{}", name, latest_version, - path_for_version(&latest_path, &krate.doc_targets, &conn, &config) + path_for_version(&latest_path, &krate.doc_targets, &storage, &config) ) } else { format!("/crate/{}/{}", name, latest_version) @@ -400,11 +401,11 @@ pub fn rustdoc_html_server_handler(req: &mut Request) -> IronResult { fn path_for_version( req_path: &[&str], known_platforms: &[String], - conn: &Connection, + storage: &Storage, config: &Config, ) -> String { // Simple case: page exists in the latest version, so just change the version number - if File::from_path(&conn, &req_path.join("/"), config).is_ok() { + if File::from_path(storage, &req_path.join("/"), config).is_ok() { // NOTE: this adds 'index.html' if it wasn't there before return req_path[3..].join("/"); } @@ -440,7 +441,9 @@ pub fn target_redirect_handler(req: &mut Request) -> IronResult { let name = cexpect!(req, router.find("name")); let version = cexpect!(req, router.find("version")); - let conn = extension!(req, Pool).get()?; + let pool = extension!(req, Pool); + let conn = pool.get()?; + let storage = extension!(req, Storage); let config = extension!(req, Config); let base = redirect_base(req); @@ -466,7 +469,7 @@ pub fn target_redirect_handler(req: &mut Request) -> IronResult { file_path }; - let path = path_for_version(&file_path, &crate_details.doc_targets, &conn, &config); + let path = path_for_version(&file_path, &crate_details.doc_targets, &storage, &config); let url = format!( "{base}/{name}/{version}/{path}", base = base, @@ -566,10 +569,10 @@ impl Handler for SharedResourceHandler { let filename = path.last().unwrap(); // unwrap is fine: vector is non-empty let suffix = filename.split('.').last().unwrap(); // unwrap is fine: split always works if ["js", "css", "woff", "svg"].contains(&suffix) { - let conn = extension!(req, Pool).get()?; + let storage = extension!(req, Storage); let config = extension!(req, Config); - if let Ok(file) = File::from_path(&conn, filename, &config) { + if let Ok(file) = File::from_path(&storage, filename, &config) { return Ok(file.serve()); } } @@ -617,9 +620,8 @@ mod test { // regression test for https://github.com/rust-lang/docs.rs/issues/552 fn settings_html() { wrapper(|env| { - let db = env.db(); // first release works, second fails - db.fake_release() + env.fake_release() .name("buggy") .version("0.1.0") .build_result_successful(true) @@ -630,7 +632,7 @@ mod test { .rustdoc_file("directory_3/.gitignore", b"*.ext") .rustdoc_file("directory_4/empty_file_no_ext", b"") .create()?; - db.fake_release() + env.fake_release() .name("buggy") .version("0.2.0") .build_result_successful(false) @@ -651,8 +653,7 @@ mod test { #[test] fn default_target_redirects_to_base() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("dummy") .version("0.1.0") .rustdoc_file("dummy/index.html", b"some content") @@ -666,7 +667,7 @@ mod test { // set an explicit target that requires cross-compile let target = "x86_64-pc-windows-msvc"; - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.0") .rustdoc_file("dummy/index.html", b"some content") @@ -679,7 +680,7 @@ mod test { // set an explicit target without cross-compile // also check that /:crate/:version/:platform/all.html doesn't panic let target = "x86_64-unknown-linux-gnu"; - db.fake_release() + env.fake_release() .name("dummy") .version("0.3.0") .rustdoc_file("dummy/index.html", b"some content") @@ -703,15 +704,14 @@ mod test { #[test] fn go_to_latest_version() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("dummy") .version("0.1.0") .rustdoc_file("dummy/blah/index.html", b"lah") .rustdoc_file("dummy/blah/blah.html", b"lah") .rustdoc_file("dummy/struct.will-be-deleted.html", b"lah") .create()?; - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.0") .rustdoc_file("dummy/blah/index.html", b"lah") @@ -749,14 +749,13 @@ mod test { #[test] fn go_to_latest_version_keeps_platform() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("dummy") .version("0.1.0") .add_platform("x86_64-pc-windows-msvc") .rustdoc_file("dummy/struct.Blah.html", b"lah") .create()?; - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.0") .add_platform("x86_64-pc-windows-msvc") @@ -797,13 +796,12 @@ mod test { #[test] fn redirect_latest_goes_to_crate_if_build_failed() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("dummy") .version("0.1.0") .rustdoc_file("dummy/index.html", b"lah") .create()?; - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.0") .build_result_successful(false) @@ -820,18 +818,17 @@ mod test { #[test] fn redirect_latest_does_not_go_to_yanked_versions() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("dummy") .version("0.1.0") .rustdoc_file("dummy/index.html", b"lah") .create()?; - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.0") .rustdoc_file("dummy/index.html", b"lah") .create()?; - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.1") .rustdoc_file("dummy/index.html", b"lah") @@ -852,20 +849,19 @@ mod test { #[test] fn redirect_latest_with_all_yanked() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("dummy") .version("0.1.0") .rustdoc_file("dummy/index.html", b"lah") .yanked(true) .create()?; - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.0") .rustdoc_file("dummy/index.html", b"lah") .yanked(true) .create()?; - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.1") .rustdoc_file("dummy/index.html", b"lah") @@ -896,9 +892,9 @@ mod test { } wrapper(|env| { - let (db, web) = (env.db(), env.frontend()); + let web = env.frontend(); - db.fake_release() + env.fake_release() .name("dummy") .version("0.1.0") .rustdoc_file("dummy/index.html", b"lah") @@ -907,7 +903,7 @@ mod test { assert!(has_yanked_warning("/dummy/0.1.0/dummy/", web)?); - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.0") .rustdoc_file("dummy/index.html", b"lah") @@ -923,8 +919,7 @@ mod test { #[test] fn badges_are_urlencoded() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("zstd") .version("0.5.1+zstd.1.4.4") .create()?; @@ -942,8 +937,7 @@ mod test { #[test] fn crate_name_percent_decoded_redirect() { wrapper(|env| { - env.db() - .fake_release() + env.fake_release() .name("fake-crate") .version("0.0.1") .rustdoc_file("fake_crate/index.html", b"some content") @@ -959,8 +953,6 @@ mod test { #[test] fn base_redirect_handles_mismatched_separators() { wrapper(|env| { - let db = env.db(); - let rels = [ ("dummy-dash", "0.1.0"), ("dummy-dash", "0.2.0"), @@ -971,7 +963,7 @@ mod test { ]; for (name, version) in &rels { - db.fake_release() + env.fake_release() .name(name) .version(version) .rustdoc_file(&(name.replace("-", "_") + "/index.html"), b"") @@ -1021,15 +1013,13 @@ mod test { #[test] fn specific_pages_do_not_handle_mismatched_separators() { wrapper(|env| { - let db = env.db(); - - db.fake_release() + env.fake_release() .name("dummy-dash") .version("0.1.0") .rustdoc_file("dummy_dash/index.html", b"") .create()?; - db.fake_release() + env.fake_release() .name("dummy_mixed-separators") .version("0.1.0") .rustdoc_file("dummy_mixed_separators/index.html", b"") @@ -1132,10 +1122,10 @@ mod test { } wrapper(|env| { - let (db, web) = (env.db(), env.frontend()); + let web = env.frontend(); // no explicit default-target - db.fake_release() + env.fake_release() .name("dummy") .version("0.1.0") .rustdoc_file("dummy/index.html", b"some content") @@ -1165,7 +1155,7 @@ mod test { )?; // set an explicit target that requires cross-compile - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.0") .rustdoc_file("dummy/index.html", b"some content") @@ -1195,7 +1185,7 @@ mod test { )?; // set an explicit target without cross-compile - db.fake_release() + env.fake_release() .name("dummy") .version("0.3.0") .rustdoc_file("dummy/index.html", b"some content") @@ -1225,7 +1215,7 @@ mod test { )?; // multiple targets - db.fake_release() + env.fake_release() .name("dummy") .version("0.4.0") .rustdoc_file("settings.html", b"top-level items") @@ -1375,9 +1365,7 @@ mod test { #[test] fn test_fully_yanked_crate_404s() { wrapper(|env| { - let db = env.db(); - - db.fake_release() + env.fake_release() .name("dummy") .version("1.0.0") .yanked(true) @@ -1401,15 +1389,14 @@ mod test { // regression test for https://github.com/rust-lang/docs.rs/issues/856 fn test_no_trailing_slash() { wrapper(|env| { - let db = env.db(); - db.fake_release().name("dummy").version("0.1.0").create()?; + env.fake_release().name("dummy").version("0.1.0").create()?; let web = env.frontend(); assert_redirect( "/crate/dummy/0.1.0/target-redirect/x86_64-apple-darwin", "/dummy/0.1.0/dummy/", web, )?; - db.fake_release() + env.fake_release() .name("dummy") .version("0.2.0") .add_platform("x86_64-apple-darwin") @@ -1433,7 +1420,11 @@ mod test { fn test_no_panic_on_missing_kind() { wrapper(|env| { let db = env.db(); - let id = db.fake_release().name("strum").version("0.13.0").create()?; + let id = env + .fake_release() + .name("strum") + .version("0.13.0") + .create()?; // https://stackoverflow.com/questions/18209625/how-do-i-modify-fields-inside-the-new-postgresql-json-datatype db.conn().query( r#"UPDATE releases SET dependencies = dependencies::jsonb #- '{0,2}' WHERE id = $1"#, @@ -1450,9 +1441,8 @@ mod test { // regression test for https://github.com/rust-lang/docs.rs/pull/885#issuecomment-655154405 fn test_readme_rendered_as_html() { wrapper(|env| { - let db = env.db(); let readme = "# Overview"; - db.fake_release() + env.fake_release() .name("strum") .version("0.18.0") .readme(readme) @@ -1473,12 +1463,11 @@ mod test { // regression test for https://github.com/rust-lang/docs.rs/pull/885#issuecomment-655149288 fn test_build_status_is_accurate() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("hexponent") .version("0.3.0") .create()?; - db.fake_release() + env.fake_release() .name("hexponent") .version("0.2.0") .build_result_successful(false) diff --git a/src/web/sitemap.rs b/src/web/sitemap.rs index c612528a6..6446e09d7 100644 --- a/src/web/sitemap.rs +++ b/src/web/sitemap.rs @@ -97,9 +97,8 @@ mod tests { let web = env.frontend(); assert_success("/sitemap.xml", web)?; - env.db().fake_release().name("some_random_crate").create()?; - env.db() - .fake_release() + env.fake_release().name("some_random_crate").create()?; + env.fake_release() .name("some_random_crate_that_failed") .build_result_successful(false) .create()?; diff --git a/src/web/source.rs b/src/web/source.rs index 912f78c5f..100d4b3d6 100644 --- a/src/web/source.rs +++ b/src/web/source.rs @@ -4,7 +4,7 @@ use crate::{ db::Pool, impl_webpage, web::{error::Nope, file::File as DbFile, page::WebPage, MetaData}, - Config, + Config, Storage, }; use iron::{status::Status, IronError, IronResult, Request, Response}; use postgres::Connection; @@ -184,13 +184,15 @@ pub fn source_browser_handler(req: &mut Request) -> IronResult { (path, file_path) }; - let conn = extension!(req, Pool).get()?; + let pool = extension!(req, Pool); + let conn = pool.get()?; + let storage = extension!(req, Storage); let config = extension!(req, Config); // try to get actual file first // skip if request is a directory let file = if !file_path.ends_with('/') { - DbFile::from_path(&conn, &file_path, &config).ok() + DbFile::from_path(&storage, &file_path, &config).ok() } else { None }; @@ -230,8 +232,7 @@ mod tests { #[test] fn cargo_ok_not_skipped() { wrapper(|env| { - let db = env.db(); - db.fake_release() + env.fake_release() .name("fake") .version("0.1.0") .source_file(".cargo-ok", b"ok")