diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index f31719bfd..7794dcf4e 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -4,11 +4,9 @@ use std::path::PathBuf; use std::sync::Arc; use cratesfyi::db::{self, add_path_into_database, Pool, PoolClient}; -use cratesfyi::index::Index; use cratesfyi::utils::{remove_crate_priority, set_crate_priority}; use cratesfyi::{ - BuildQueue, Config, Context, DocBuilder, DocBuilderOptions, Metrics, RustwideBuilder, Server, - Storage, + BuildQueue, Config, Context, DocBuilder, Index, Metrics, RustwideBuilder, Server, Storage, }; use failure::{err_msg, Error, ResultExt}; use once_cell::sync::OnceCell; @@ -230,41 +228,13 @@ struct Build { #[structopt(name = "SKIP_IF_EXISTS", short = "s", long = "skip")] skip_if_exists: bool, - /// Skips building documentation if build log exists - #[structopt(name = "SKIP_IF_LOG_EXISTS", long = "skip-if-log-exists")] - skip_if_log_exists: bool, - - /// Keeps build directory after build. - #[structopt( - name = "KEEP_BUILD_DIRECTORY", - short = "k", - long = "keep-build-directory" - )] - keep_build_directory: bool, - #[structopt(subcommand)] subcommand: BuildSubcommand, } impl Build { pub fn handle_args(self, ctx: BinContext) -> Result<(), Error> { - let docbuilder = { - let config = ctx.config()?; - let mut doc_options = - DocBuilderOptions::new(config.prefix.clone(), config.registry_index_path.clone()); - - doc_options.skip_if_exists = self.skip_if_exists; - doc_options.skip_if_log_exists = self.skip_if_log_exists; - doc_options.keep_build_directory = self.keep_build_directory; - - doc_options - .check_paths() - .context("The given paths were invalid")?; - - DocBuilder::new(doc_options, ctx.pool()?, ctx.build_queue()?) - }; - - self.subcommand.handle_args(ctx, docbuilder) + self.subcommand.handle_args(ctx, self.skip_if_exists) } } @@ -307,23 +277,23 @@ enum BuildSubcommand { /// Unlocks cratesfyi daemon to continue building new crates Unlock, - - PrintOptions, } impl BuildSubcommand { - pub fn handle_args(self, ctx: BinContext, mut docbuilder: DocBuilder) -> Result<(), Error> { + pub fn handle_args(self, ctx: BinContext, skip_if_exists: bool) -> Result<(), Error> { + let docbuilder = DocBuilder::new(ctx.config()?, ctx.pool()?, ctx.build_queue()?); + + let rustwide_builder = || -> Result { + let mut builder = RustwideBuilder::init(&ctx)?; + builder.set_skip_build_if_exists(skip_if_exists); + Ok(builder) + }; + match self { Self::World => { - docbuilder.load_cache().context("Failed to load cache")?; - - let mut builder = - RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?)?; - builder - .build_world(&mut docbuilder) + rustwide_builder()? + .build_world() .context("Failed to build world")?; - - docbuilder.save_cache().context("Failed to save cache")?; } Self::Crate { @@ -331,19 +301,15 @@ impl BuildSubcommand { crate_version, local, } => { - docbuilder.load_cache().context("Failed to load cache")?; - let mut builder = - RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?) - .context("failed to initialize rustwide")?; + let mut builder = rustwide_builder()?; if let Some(path) = local { builder - .build_local_package(&mut docbuilder, &path) + .build_local_package(&path) .context("Building documentation failed")?; } else { builder .build_package( - &mut docbuilder, &crate_name.ok_or_else(|| err_msg("must specify name if not local"))?, &crate_version .ok_or_else(|| err_msg("must specify version if not local"))?, @@ -351,8 +317,6 @@ impl BuildSubcommand { ) .context("Building documentation failed")?; } - - docbuilder.save_cache().context("Failed to save cache")?; } Self::UpdateToolchain { only_first_time } => { @@ -370,24 +334,19 @@ impl BuildSubcommand { } } - let mut builder = - RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?)?; - builder + rustwide_builder()? .update_toolchain() .context("failed to update toolchain")?; } Self::AddEssentialFiles => { - let mut builder = - RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?)?; - builder + rustwide_builder()? .add_essential_files() .context("failed to add essential files")?; } Self::Lock => docbuilder.lock().context("Failed to lock")?, Self::Unlock => docbuilder.unlock().context("Failed to unlock")?, - Self::PrintOptions => println!("{:?}", docbuilder.options()), } Ok(()) @@ -458,7 +417,7 @@ impl DatabaseSubcommand { } Self::UpdateCrateRegistryFields { name } => { - let index = Index::new(&ctx.config()?.registry_index_path)?; + let index = ctx.index()?; db::update_crate_data_in_database( &mut *ctx.conn()?, @@ -490,8 +449,8 @@ impl DatabaseSubcommand { Self::Synchronize { dry_run } => { cratesfyi::utils::consistency::run_check( - &*ctx.config()?, &mut *ctx.conn()?, + &*ctx.index()?, dry_run, )?; } @@ -567,6 +526,7 @@ struct BinContext { config: OnceCell>, pool: OnceCell, metrics: OnceCell>, + index: OnceCell>, } impl BinContext { @@ -577,6 +537,7 @@ impl BinContext { config: OnceCell::new(), pool: OnceCell::new(), metrics: OnceCell::new(), + index: OnceCell::new(), } } @@ -632,4 +593,11 @@ impl Context for BinContext { .get_or_try_init::<_, Error>(|| Ok(Arc::new(Metrics::new()?)))? .clone()) } + + fn index(&self) -> Result, Error> { + Ok(self + .index + .get_or_try_init::<_, Error>(|| Ok(Arc::new(Index::new(&*self.config()?)?)))? + .clone()) + } } diff --git a/src/config.rs b/src/config.rs index d84f12872..568f6bd19 100644 --- a/src/config.rs +++ b/src/config.rs @@ -32,6 +32,12 @@ pub struct Config { pub(crate) max_parse_memory: usize, // Time between 'git gc --auto' calls in seconds pub(crate) registry_gc_interval: u64, + + pub(crate) rustwide_workspace: PathBuf, + pub(crate) inside_docker: bool, + pub(crate) local_docker_image: Option, + pub(crate) toolchain: String, + pub(crate) build_cpu_limit: Option, } impl Config { @@ -64,6 +70,12 @@ impl Config { // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380 max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?, registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?, + + rustwide_workspace: env("CRATESFYI_RUSTWIDE_WORKSPACE", PathBuf::from(".workspace"))?, + inside_docker: env("DOCS_RS_DOCKER", false)?, + local_docker_image: maybe_env("DOCS_RS_LOCAL_DOCKER_IMAGE")?, + toolchain: env("CRATESFYI_TOOLCHAIN", "nightly".to_string())?, + build_cpu_limit: maybe_env("DOCS_RS_BUILD_CPU_LIMIT")?, }) } diff --git a/src/context.rs b/src/context.rs index cc18b1430..f0bfa1aff 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,5 +1,5 @@ use crate::db::Pool; -use crate::{BuildQueue, Config, Metrics, Storage}; +use crate::{BuildQueue, Config, Index, Metrics, Storage}; use failure::Error; use std::sync::Arc; @@ -9,4 +9,5 @@ pub trait Context { fn storage(&self) -> Result, Error>; fn pool(&self) -> Result; fn metrics(&self) -> Result, Error>; + fn index(&self) -> Result, Error>; } diff --git a/src/docbuilder/mod.rs b/src/docbuilder/mod.rs index 220ad062e..568571255 100644 --- a/src/docbuilder/mod.rs +++ b/src/docbuilder/mod.rs @@ -1,6 +1,5 @@ mod crates; mod limits; -pub(crate) mod options; mod queue; mod rustwide_builder; @@ -10,92 +9,29 @@ pub(crate) use self::rustwide_builder::{BuildResult, DocCoverage}; use crate::db::Pool; use crate::error::Result; -use crate::index::Index; -use crate::BuildQueue; -use crate::DocBuilderOptions; -use log::debug; -use std::collections::BTreeSet; +use crate::{BuildQueue, Config}; use std::fs; -use std::io::prelude::*; -use std::io::BufReader; use std::path::PathBuf; use std::sync::Arc; /// chroot based documentation builder pub struct DocBuilder { - options: DocBuilderOptions, - index: Index, + config: Arc, db: Pool, build_queue: Arc, - cache: BTreeSet, - db_cache: BTreeSet, } impl DocBuilder { - pub fn new(options: DocBuilderOptions, db: Pool, build_queue: Arc) -> DocBuilder { - let index = Index::new(&options.registry_index_path).expect("valid index"); + pub fn new(config: Arc, db: Pool, build_queue: Arc) -> DocBuilder { DocBuilder { + config, build_queue, - options, - index, db, - cache: BTreeSet::new(), - db_cache: BTreeSet::new(), } } - /// Loads build cache - pub fn load_cache(&mut self) -> Result<()> { - debug!("Loading cache"); - - let path = PathBuf::from(&self.options.prefix).join("cache"); - let reader = fs::File::open(path).map(BufReader::new); - - if let Ok(reader) = reader { - for line in reader.lines() { - self.cache.insert(line?); - } - } - - self.load_database_cache()?; - - Ok(()) - } - - fn load_database_cache(&mut self) -> Result<()> { - debug!("Loading database cache"); - - let mut conn = self.db.get()?; - for row in &mut conn.query( - "SELECT name, version FROM crates, releases \ - WHERE crates.id = releases.crate_id", - &[], - )? { - let name: String = row.get(0); - let version: String = row.get(1); - - self.db_cache.insert(format!("{}-{}", name, version)); - } - - Ok(()) - } - - /// Saves build cache - pub fn save_cache(&self) -> Result<()> { - debug!("Saving cache"); - - let path = PathBuf::from(&self.options.prefix).join("cache"); - let mut file = fs::OpenOptions::new().write(true).create(true).open(path)?; - - for krate in &self.cache { - writeln!(file, "{}", krate)?; - } - - Ok(()) - } - fn lock_path(&self) -> PathBuf { - self.options.prefix.join("cratesfyi.lock") + self.config.prefix.join("cratesfyi.lock") } /// Creates a lock file. Daemon will check this lock file and stop operating if its exists. @@ -122,21 +58,4 @@ impl DocBuilder { pub fn is_locked(&self) -> bool { self.lock_path().exists() } - - /// Returns a reference of options - pub fn options(&self) -> &DocBuilderOptions { - &self.options - } - - fn add_to_cache(&mut self, name: &str, version: &str) { - self.cache.insert(format!("{}-{}", name, version)); - } - - fn should_build(&self, name: &str, version: &str) -> bool { - let name = format!("{}-{}", name, version); - let local = self.options.skip_if_log_exists && self.cache.contains(&name); - let db = self.options.skip_if_exists && self.db_cache.contains(&name); - - !(local || db) - } } diff --git a/src/docbuilder/options.rs b/src/docbuilder/options.rs deleted file mode 100644 index a9cf399c8..000000000 --- a/src/docbuilder/options.rs +++ /dev/null @@ -1,41 +0,0 @@ -use crate::error::Result; -use std::path::PathBuf; - -#[derive(Clone, Debug)] -pub struct DocBuilderOptions { - pub keep_build_directory: bool, - pub prefix: PathBuf, - pub registry_index_path: PathBuf, - pub skip_if_exists: bool, - pub skip_if_log_exists: bool, - pub skip_oldest_versions: bool, - pub build_only_latest_version: bool, - pub debug: bool, -} - -impl DocBuilderOptions { - pub fn new(prefix: PathBuf, registry_index_path: PathBuf) -> DocBuilderOptions { - DocBuilderOptions { - prefix, - registry_index_path, - - keep_build_directory: false, - skip_if_exists: false, - skip_if_log_exists: false, - skip_oldest_versions: false, - build_only_latest_version: false, - debug: false, - } - } - - pub fn check_paths(&self) -> Result<()> { - if !self.registry_index_path.exists() { - failure::bail!( - "registry index path '{}' does not exist", - self.registry_index_path.display() - ); - } - - Ok(()) - } -} diff --git a/src/docbuilder/queue.rs b/src/docbuilder/queue.rs index d357fd2d6..5dfb674b0 100644 --- a/src/docbuilder/queue.rs +++ b/src/docbuilder/queue.rs @@ -3,15 +3,16 @@ use super::{DocBuilder, RustwideBuilder}; use crate::error::Result; use crate::utils::get_crate_priority; +use crate::Index; use crates_index_diff::ChangeKind; use log::{debug, error}; impl DocBuilder { /// Updates registry index repository and adds new crates into build queue. /// Returns the number of crates added - pub fn get_new_crates(&mut self) -> Result { + pub fn get_new_crates(&mut self, index: &Index) -> Result { let mut conn = self.db.get()?; - let diff = self.index.diff()?; + let diff = index.diff()?; let (mut changes, oid) = diff.peek_changes()?; let mut crates_added = 0; @@ -78,14 +79,10 @@ impl DocBuilder { queue.process_next_crate(|krate| { processed = true; - builder.build_package(self, &krate.name, &krate.version, None)?; + builder.build_package(&krate.name, &krate.version, None)?; Ok(()) })?; Ok(processed) } - - pub fn run_git_gc(&self) { - self.index.run_git_gc(); - } } diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index d9b4c8942..925a7cab7 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -1,4 +1,3 @@ -use super::DocBuilder; use crate::db::blacklist::is_blacklisted; use crate::db::file::add_path_into_database; use crate::db::{ @@ -10,22 +9,21 @@ use crate::error::Result; use crate::index::api::ReleaseData; use crate::storage::CompressionAlgorithms; use crate::utils::{copy_doc_dir, parse_rustc_version, CargoMetadata}; -use crate::{Metrics, Storage}; +use crate::{Config, Context, Index, Metrics, Storage}; use docsrs_metadata::{Metadata, DEFAULT_TARGETS, HOST_TARGET}; use failure::ResultExt; use log::{debug, info, warn, LevelFilter}; -use rustwide::cmd::{Command, SandboxBuilder}; +use postgres::Client; +use rustwide::cmd::{Command, SandboxBuilder, SandboxImage}; use rustwide::logging::{self, LogStorage}; use rustwide::toolchain::ToolchainError; use rustwide::{Build, Crate, Toolchain, Workspace, WorkspaceBuilder}; use serde_json::Value; -use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::path::Path; use std::sync::Arc; const USER_AGENT: &str = "docs.rs builder (https://github.com/rust-lang/docs.rs)"; -const DEFAULT_RUSTWIDE_WORKSPACE: &str = ".rustwide"; const ESSENTIAL_FILES_VERSIONED: &[&str] = &[ "brush.svg", "wheel.svg", @@ -60,59 +58,50 @@ const DUMMY_CRATE_VERSION: &str = "1.0.0"; pub struct RustwideBuilder { workspace: Workspace, toolchain: Toolchain, + config: Arc, db: Pool, storage: Arc, metrics: Arc, + index: Arc, rustc_version: String, - cpu_limit: Option, + skip_build_if_exists: bool, } impl RustwideBuilder { - pub fn init(db: Pool, metrics: Arc, storage: Arc) -> Result { - use rustwide::cmd::SandboxImage; - let env_workspace_path = ::std::env::var("CRATESFYI_RUSTWIDE_WORKSPACE"); - let workspace_path = env_workspace_path - .as_ref() - .map(|v| v.as_str()) - .unwrap_or(DEFAULT_RUSTWIDE_WORKSPACE); - let is_docker = std::env::var("DOCS_RS_DOCKER") - .map(|s| s == "true") - .unwrap_or(false); - let mut builder = WorkspaceBuilder::new(Path::new(workspace_path), USER_AGENT) - .running_inside_docker(is_docker); - if let Ok(custom_image) = std::env::var("DOCS_RS_LOCAL_DOCKER_IMAGE") { + pub fn init(context: &dyn Context) -> Result { + let config = context.config()?; + + let mut builder = WorkspaceBuilder::new(&config.rustwide_workspace, USER_AGENT) + .running_inside_docker(config.inside_docker); + if let Some(custom_image) = &config.local_docker_image { builder = builder.sandbox_image(SandboxImage::local(&custom_image)?); } let workspace = builder.init()?; workspace.purge_all_build_dirs()?; - let toolchain_name = std::env::var("CRATESFYI_TOOLCHAIN") - .map(Cow::Owned) - .unwrap_or_else(|_| Cow::Borrowed("nightly")); - - let cpu_limit = std::env::var("DOCS_RS_BUILD_CPU_LIMIT").ok().map(|limit| { - limit - .parse::() - .expect("invalid DOCS_RS_BUILD_CPU_LIMIT") - }); - - let toolchain = Toolchain::dist(&toolchain_name); + let toolchain = Toolchain::dist(&config.toolchain); Ok(RustwideBuilder { workspace, toolchain, - db, - storage, - metrics, + config, + db: context.pool()?, + storage: context.storage()?, + metrics: context.metrics()?, + index: context.index()?, rustc_version: String::new(), - cpu_limit, + skip_build_if_exists: false, }) } + pub fn set_skip_build_if_exists(&mut self, should: bool) { + self.skip_build_if_exists = should; + } + fn prepare_sandbox(&self, limits: &Limits) -> SandboxBuilder { SandboxBuilder::new() - .cpu_limit(self.cpu_limit.map(|limit| limit as f32)) + .cpu_limit(self.config.build_cpu_limit.map(|limit| limit as f32)) .memory_limit(Some(limits.memory())) .enable_networking(limits.networking()) } @@ -265,47 +254,36 @@ impl RustwideBuilder { Ok(()) } - pub fn build_world(&mut self, doc_builder: &mut DocBuilder) -> Result<()> { - let mut count = 0; + pub fn build_world(&mut self) -> Result<()> { crates_from_path( - &doc_builder.options().registry_index_path.clone(), + &self.config.registry_index_path.clone(), &mut |name, version| { - match self.build_package(doc_builder, name, version, None) { - Ok(status) => { - count += 1; - if status && count % 10 == 0 { - let _ = doc_builder.save_cache(); - } - } - Err(err) => warn!("failed to build package {} {}: {}", name, version, err), + if let Err(err) = self.build_package(name, version, None) { + warn!("failed to build package {} {}: {}", name, version, err); } - doc_builder.add_to_cache(name, version); }, ) } - pub fn build_local_package( - &mut self, - doc_builder: &mut DocBuilder, - path: &Path, - ) -> Result { + pub fn build_local_package(&mut self, path: &Path) -> Result { self.update_toolchain()?; let metadata = CargoMetadata::load(&self.workspace, &self.toolchain, path).map_err(|err| { err.context(format!("failed to load local package {}", path.display())) })?; let package = metadata.root(); - self.build_package(doc_builder, &package.name, &package.version, Some(path)) + self.build_package(&package.name, &package.version, Some(path)) } pub fn build_package( &mut self, - doc_builder: &mut DocBuilder, name: &str, version: &str, local: Option<&Path>, ) -> Result { - if !doc_builder.should_build(name, version) { + let mut conn = self.db.get()?; + + if !self.should_build(&mut conn, name, version)? { return Ok(false); } @@ -313,8 +291,6 @@ impl RustwideBuilder { info!("building package {} {}", name, version); - let mut conn = self.db.get()?; - if is_blacklisted(&mut conn, name)? { info!("skipping build of {}, crate has been blacklisted", name); return Ok(false); @@ -396,7 +372,7 @@ impl RustwideBuilder { self.metrics.non_library_builds.inc(); } - let release_data = match doc_builder.index.api().get_release_data(name, version) { + let release_data = match self.index.api().get_release_data(name, version) { Ok(data) => data, Err(err) => { warn!("{:#?}", err); @@ -425,12 +401,11 @@ impl RustwideBuilder { add_build_into_database(&mut conn, release_id, &res.result)?; // Some crates.io crate data is mutable, so we proactively update it during a release - match doc_builder.index.api().get_crate_data(name) { + match self.index.api().get_crate_data(name) { Ok(crate_data) => update_crate_data_in_database(&mut conn, name, &crate_data)?, Err(err) => warn!("{:#?}", err), } - doc_builder.add_to_cache(name, version); Ok(res) })?; @@ -596,7 +571,7 @@ impl RustwideBuilder { let mut cargo_args = metadata.cargo_args(); // Add docs.rs specific arguments - if let Some(cpu_limit) = self.cpu_limit { + if let Some(cpu_limit) = self.config.build_cpu_limit { cargo_args.push(format!("-j{}", cpu_limit)); } if target != HOST_TARGET { @@ -657,6 +632,23 @@ impl RustwideBuilder { ) .map(|t| t.1) } + + fn should_build(&self, conn: &mut Client, name: &str, version: &str) -> Result { + if self.skip_build_if_exists { + // Check whether no successful builds are present in the database. + Ok(conn + .query( + "SELECT 1 FROM crates, releases, builds + WHERE crates.id = releases.crate_id AND releases.id = builds.rid + AND crates.name = $1 AND releases.version = $2 + AND builds.build_status = TRUE;", + &[&name, &version], + )? + .is_empty()) + } else { + Ok(true) + } + } } struct FullBuildResult { diff --git a/src/index/mod.rs b/src/index/mod.rs index c6ca75869..ec415fd49 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -1,12 +1,9 @@ -use std::{ - path::{Path, PathBuf}, - process::Command, -}; +use std::{path::PathBuf, process::Command}; use url::Url; use self::{api::Api, crates::Crates}; -use crate::error::Result; +use crate::{error::Result, Config}; use failure::ResultExt; pub(crate) mod api; @@ -43,8 +40,8 @@ fn load_config(repo: &git2::Repository) -> Result { } impl Index { - pub fn new(path: impl AsRef) -> Result { - let path = path.as_ref().to_owned(); + pub fn new(app_config: &Config) -> Result { + let path = app_config.registry_index_path.clone(); // This initializes the repository, then closes it afterwards to avoid leaking file descriptors. // See https://github.com/rust-lang/docs.rs/pull/847 let diff = crates_index_diff::Index::from_path_or_cloned(&path) @@ -91,9 +88,3 @@ impl Index { } } } - -impl Clone for Index { - fn clone(&self) -> Self { - Self::new(&self.path).expect("we already loaded this registry successfully once") - } -} diff --git a/src/lib.rs b/src/lib.rs index 0d4ab7798..5c3f06eea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,9 +5,9 @@ pub use self::build_queue::BuildQueue; pub use self::config::Config; pub use self::context::Context; -pub use self::docbuilder::options::DocBuilderOptions; pub use self::docbuilder::DocBuilder; pub use self::docbuilder::RustwideBuilder; +pub use self::index::Index; pub use self::metrics::Metrics; pub use self::storage::Storage; pub use self::web::Server; diff --git a/src/test/mod.rs b/src/test/mod.rs index 2a6d7cdfb..c147a72c4 100644 --- a/src/test/mod.rs +++ b/src/test/mod.rs @@ -3,7 +3,7 @@ mod fakes; use crate::db::{Pool, PoolClient}; use crate::storage::Storage; use crate::web::Server; -use crate::{BuildQueue, Config, Context, Metrics}; +use crate::{BuildQueue, Config, Context, Index, Metrics}; use failure::Error; use log::error; use once_cell::unsync::OnceCell; @@ -96,6 +96,7 @@ pub(crate) struct TestEnvironment { config: OnceCell>, db: OnceCell, storage: OnceCell>, + index: OnceCell>, metrics: OnceCell>, frontend: OnceCell, s3: OnceCell>, @@ -117,6 +118,7 @@ impl TestEnvironment { config: OnceCell::new(), db: OnceCell::new(), storage: OnceCell::new(), + index: OnceCell::new(), metrics: OnceCell::new(), frontend: OnceCell::new(), s3: OnceCell::new(), @@ -193,6 +195,14 @@ impl TestEnvironment { .clone() } + pub(crate) fn index(&self) -> Arc { + self.index + .get_or_init(|| { + Arc::new(Index::new(&*self.config()).expect("failed to initialize the index")) + }) + .clone() + } + pub(crate) fn db(&self) -> &TestDatabase { self.db.get_or_init(|| { TestDatabase::new(&self.config(), self.metrics()).expect("failed to initialize the db") @@ -250,6 +260,10 @@ impl Context for TestEnvironment { fn metrics(&self) -> Result, Error> { Ok(self.metrics()) } + + fn index(&self) -> Result, Error> { + Ok(self.index()) + } } pub(crate) struct TestDatabase { diff --git a/src/utils/consistency/index.rs b/src/utils/consistency/index.rs index 0517c21c0..4b0a30360 100644 --- a/src/utils/consistency/index.rs +++ b/src/utils/consistency/index.rs @@ -1,9 +1,7 @@ use super::data::{Crate, CrateName, Data, Release, Version}; -use crate::{config::Config, index::Index}; - -pub(crate) fn load(config: &Config) -> Result { - let index = Index::new(&config.registry_index_path)?; +use crate::Index; +pub(crate) fn load(index: &Index) -> Result { let mut data = Data::default(); index.crates()?.walk(|krate| { diff --git a/src/utils/consistency/mod.rs b/src/utils/consistency/mod.rs index 1c8f128e8..5ee568b3b 100644 --- a/src/utils/consistency/mod.rs +++ b/src/utils/consistency/mod.rs @@ -1,5 +1,5 @@ use self::diff::{Diff, Diffable}; -use crate::config::Config; +use crate::Index; use failure::ResultExt; mod data; @@ -8,8 +8,8 @@ mod diff; mod index; pub fn run_check( - config: &Config, conn: &mut postgres::Client, + index: &Index, dry_run: bool, ) -> Result<(), failure::Error> { if !dry_run { @@ -25,7 +25,7 @@ pub fn run_check( log::info!("Loading data from index..."); let timer = std::time::Instant::now(); let index_data = - self::index::load(config).context("Loading crate data from index for consistency check")?; + self::index::load(index).context("Loading crate data from index for consistency check")?; log::info!("...loaded in {:?}", timer.elapsed()); let diff = db_data.diff(index_data); diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 61a9fa062..efe98e3a5 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -4,7 +4,7 @@ use crate::{ utils::{queue_builder, update_release_activity, GithubUpdater}, - Context, DocBuilder, DocBuilderOptions, + Context, DocBuilder, RustwideBuilder, }; use chrono::{Timelike, Utc}; use failure::Error; @@ -12,10 +12,11 @@ use log::{debug, error, info}; use std::thread; use std::time::{Duration, Instant}; -fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Result<(), Error> { +fn start_registry_watcher(context: &dyn Context) -> Result<(), Error> { let pool = context.pool()?; let build_queue = context.build_queue()?; let config = context.config()?; + let index = context.index()?; thread::Builder::new() .name("registry index reader".to_string()) @@ -26,20 +27,20 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res let mut last_gc = Instant::now(); loop { let mut doc_builder = - DocBuilder::new(opts.clone(), pool.clone(), build_queue.clone()); + DocBuilder::new(config.clone(), pool.clone(), build_queue.clone()); if doc_builder.is_locked() { debug!("Lock file exists, skipping checking new crates"); } else { debug!("Checking new crates"); - match doc_builder.get_new_crates() { + match doc_builder.get_new_crates(&index) { Ok(n) => debug!("{} crates added to queue", n), Err(e) => error!("Failed to get new crates: {}", e), } } if last_gc.elapsed().as_secs() >= config.registry_gc_interval { - doc_builder.run_git_gc(); + index.run_git_gc(); last_gc = Instant::now(); } thread::sleep(Duration::from_secs(60)); @@ -51,26 +52,23 @@ fn start_registry_watcher(opts: DocBuilderOptions, context: &dyn Context) -> Res pub fn start_daemon(context: &dyn Context, enable_registry_watcher: bool) -> Result<(), Error> { let config = context.config()?; - let dbopts = DocBuilderOptions::new(config.prefix.clone(), config.registry_index_path.clone()); - - // check paths once - dbopts.check_paths().unwrap(); if enable_registry_watcher { // check new crates every minute - start_registry_watcher(dbopts.clone(), context)?; + start_registry_watcher(context)?; } // build new crates every minute let pool = context.pool()?; let build_queue = context.build_queue()?; - let storage = context.storage()?; - let metrics = context.metrics()?; + let cloned_config = config.clone(); + let rustwide_builder = RustwideBuilder::init(context)?; thread::Builder::new() .name("build queue reader".to_string()) .spawn(move || { - let doc_builder = DocBuilder::new(dbopts.clone(), pool.clone(), build_queue.clone()); - queue_builder(doc_builder, pool, build_queue, metrics, storage).unwrap(); + let doc_builder = + DocBuilder::new(cloned_config.clone(), pool.clone(), build_queue.clone()); + queue_builder(doc_builder, rustwide_builder, build_queue).unwrap(); }) .unwrap(); diff --git a/src/utils/queue_builder.rs b/src/utils/queue_builder.rs index 95f5285cf..07cc4edbe 100644 --- a/src/utils/queue_builder.rs +++ b/src/utils/queue_builder.rs @@ -1,7 +1,4 @@ -use crate::{ - db::Pool, docbuilder::RustwideBuilder, utils::pubsubhubbub, BuildQueue, DocBuilder, Metrics, - Storage, -}; +use crate::{docbuilder::RustwideBuilder, utils::pubsubhubbub, BuildQueue, DocBuilder}; use failure::Error; use log::{debug, error, info, warn}; use std::panic::{catch_unwind, AssertUnwindSafe}; @@ -12,10 +9,8 @@ use std::time::Duration; // TODO: change to `fn() -> Result` when never _finally_ stabilizes pub fn queue_builder( mut doc_builder: DocBuilder, - db: Pool, + mut builder: RustwideBuilder, build_queue: Arc, - metrics: Arc, - storage: Arc, ) -> Result<(), Error> { /// Represents the current state of the builder thread. enum BuilderState { @@ -30,8 +25,6 @@ pub fn queue_builder( QueueInProgress(usize), } - let mut builder = RustwideBuilder::init(db, metrics, storage)?; - let mut status = BuilderState::Fresh; loop { @@ -47,22 +40,14 @@ pub fn queue_builder( } if status.count() >= 10 { - // periodically, we need to flush our caches and ping the hubs - debug!("10 builds in a row; flushing caches"); + // periodically, ping the hubs + debug!("10 builds in a row; pinging pubsubhubhub"); status = BuilderState::QueueInProgress(0); match pubsubhubbub::ping_hubs() { Err(e) => error!("Failed to ping hub: {}", e), Ok(n) => debug!("Succesfully pinged {} hubs", n), } - - if let Err(e) = doc_builder.load_cache() { - error!("Failed to load cache: {}", e); - } - - if let Err(e) = doc_builder.save_cache() { - error!("Failed to save cache: {}", e); - } } // Only build crates if there are any to build @@ -80,10 +65,6 @@ pub fn queue_builder( Err(e) => error!("Failed to ping hub: {}", e), Ok(n) => debug!("Succesfully pinged {} hubs", n), } - - if let Err(e) = doc_builder.save_cache() { - error!("Failed to save cache: {}", e); - } } debug!("Queue is empty, going back to sleep"); status = BuilderState::EmptyQueue; @@ -99,14 +80,6 @@ pub fn queue_builder( } } - // if we're starting a new batch, reload our caches and sources - if !status.is_in_progress() { - if let Err(e) = doc_builder.load_cache() { - error!("Failed to load cache: {}", e); - continue; - } - } - // Run build_packages_queue under `catch_unwind` to catch panics // This only panicked twice in the last 6 months but its just a better // idea to do this.