Remove DocBuilder, part 1

src/bin/cratesfyi.rs

@@ -4,11 +4,9 @@ use std::path::PathBuf;
 use std::sync::Arc;
 
 use cratesfyi::db::{self, add_path_into_database, Pool, PoolClient};
-use cratesfyi::index::Index;
 use cratesfyi::utils::{remove_crate_priority, set_crate_priority};
 use cratesfyi::{
-    BuildQueue, Config, Context, DocBuilder, DocBuilderOptions, Metrics, RustwideBuilder, Server,
-    Storage,
+    BuildQueue, Config, Context, DocBuilder, Index, Metrics, RustwideBuilder, Server, Storage,
 };
 use failure::{err_msg, Error, ResultExt};
 use once_cell::sync::OnceCell;
@@ -230,41 +228,13 @@ struct Build {
     #[structopt(name = "SKIP_IF_EXISTS", short = "s", long = "skip")]
     skip_if_exists: bool,
 
-    /// Skips building documentation if build log exists
-    #[structopt(name = "SKIP_IF_LOG_EXISTS", long = "skip-if-log-exists")]
-    skip_if_log_exists: bool,
-
-    /// Keeps build directory after build.
-    #[structopt(
-        name = "KEEP_BUILD_DIRECTORY",
-        short = "k",
-        long = "keep-build-directory"
-    )]
-    keep_build_directory: bool,
-
     #[structopt(subcommand)]
     subcommand: BuildSubcommand,
 }
 
 impl Build {
     pub fn handle_args(self, ctx: BinContext) -> Result<(), Error> {
-        let docbuilder = {
-            let config = ctx.config()?;
-            let mut doc_options =
-                DocBuilderOptions::new(config.prefix.clone(), config.registry_index_path.clone());
-
-            doc_options.skip_if_exists = self.skip_if_exists;
-            doc_options.skip_if_log_exists = self.skip_if_log_exists;
-            doc_options.keep_build_directory = self.keep_build_directory;
-
-            doc_options
-                .check_paths()
-                .context("The given paths were invalid")?;
-
-            DocBuilder::new(doc_options, ctx.pool()?, ctx.build_queue()?)
-        };
-
-        self.subcommand.handle_args(ctx, docbuilder)
+        self.subcommand.handle_args(ctx, self.skip_if_exists)
     }
 }
 
@@ -307,52 +277,46 @@ enum BuildSubcommand {
 
     /// Unlocks cratesfyi daemon to continue building new crates
     Unlock,
-
-    PrintOptions,
 }
 
 impl BuildSubcommand {
-    pub fn handle_args(self, ctx: BinContext, mut docbuilder: DocBuilder) -> Result<(), Error> {
+    pub fn handle_args(self, ctx: BinContext, skip_if_exists: bool) -> Result<(), Error> {
+        let docbuilder = DocBuilder::new(ctx.config()?, ctx.pool()?, ctx.build_queue()?);
+
+        let rustwide_builder = || -> Result<RustwideBuilder, Error> {
+            let mut builder = RustwideBuilder::init(&ctx)?;
+            builder.set_skip_build_if_exists(skip_if_exists);
+            Ok(builder)
+        };
+
         match self {
             Self::World => {
-                docbuilder.load_cache().context("Failed to load cache")?;
-
-                let mut builder =
-                    RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?)?;
-                builder
-                    .build_world(&mut docbuilder)
+                rustwide_builder()?
+                    .build_world()
                     .context("Failed to build world")?;
-
-                docbuilder.save_cache().context("Failed to save cache")?;
             }
 
             Self::Crate {
                 crate_name,
                 crate_version,
                 local,
             } => {
-                docbuilder.load_cache().context("Failed to load cache")?;
-                let mut builder =
-                    RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?)
-                        .context("failed to initialize rustwide")?;
+                let mut builder = rustwide_builder()?;
 
                 if let Some(path) = local {
                     builder
-                        .build_local_package(&mut docbuilder, &path)
+                        .build_local_package(&path)
                         .context("Building documentation failed")?;
                 } else {
                     builder
                         .build_package(
-                            &mut docbuilder,
                             &crate_name.ok_or_else(|| err_msg("must specify name if not local"))?,
                             &crate_version
                                 .ok_or_else(|| err_msg("must specify version if not local"))?,
                             None,
                         )
                         .context("Building documentation failed")?;
                 }
-
-                docbuilder.save_cache().context("Failed to save cache")?;
             }
 
             Self::UpdateToolchain { only_first_time } => {
@@ -370,24 +334,19 @@ impl BuildSubcommand {
                     }
                 }
 
-                let mut builder =
-                    RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?)?;
-                builder
+                rustwide_builder()?
                     .update_toolchain()
                     .context("failed to update toolchain")?;
             }
 
             Self::AddEssentialFiles => {
-                let mut builder =
-                    RustwideBuilder::init(ctx.pool()?, ctx.metrics()?, ctx.storage()?)?;
-                builder
+                rustwide_builder()?
                     .add_essential_files()
                     .context("failed to add essential files")?;
             }
 
             Self::Lock => docbuilder.lock().context("Failed to lock")?,
             Self::Unlock => docbuilder.unlock().context("Failed to unlock")?,
-            Self::PrintOptions => println!("{:?}", docbuilder.options()),
         }
 
         Ok(())
@@ -458,7 +417,7 @@ impl DatabaseSubcommand {
             }
 
             Self::UpdateCrateRegistryFields { name } => {
-                let index = Index::new(&ctx.config()?.registry_index_path)?;
+                let index = ctx.index()?;
 
                 db::update_crate_data_in_database(
                     &mut *ctx.conn()?,
@@ -490,8 +449,8 @@ impl DatabaseSubcommand {
 
             Self::Synchronize { dry_run } => {
                 cratesfyi::utils::consistency::run_check(
-                    &*ctx.config()?,
                     &mut *ctx.conn()?,
+                    &*ctx.index()?,
                     dry_run,
                 )?;
             }
@@ -567,6 +526,7 @@ struct BinContext {
     config: OnceCell<Arc<Config>>,
     pool: OnceCell<Pool>,
     metrics: OnceCell<Arc<Metrics>>,
+    index: OnceCell<Arc<Index>>,
 }
 
 impl BinContext {
@@ -577,6 +537,7 @@ impl BinContext {
             config: OnceCell::new(),
             pool: OnceCell::new(),
             metrics: OnceCell::new(),
+            index: OnceCell::new(),
         }
     }
 
@@ -632,4 +593,11 @@ impl Context for BinContext {
             .get_or_try_init::<_, Error>(|| Ok(Arc::new(Metrics::new()?)))?
             .clone())
     }
+
+    fn index(&self) -> Result<Arc<Index>, Error> {
+        Ok(self
+            .index
+            .get_or_try_init::<_, Error>(|| Ok(Arc::new(Index::new(&*self.config()?)?)))?
+            .clone())
+    }
 }

src/config.rs

@@ -32,6 +32,12 @@ pub struct Config {
     pub(crate) max_parse_memory: usize,
     // Time between 'git gc --auto' calls in seconds
     pub(crate) registry_gc_interval: u64,
+
+    pub(crate) rustwide_workspace: PathBuf,
+    pub(crate) inside_docker: bool,
+    pub(crate) local_docker_image: Option<String>,
+    pub(crate) toolchain: String,
+    pub(crate) build_cpu_limit: Option<u32>,
 }
 
 impl Config {
@@ -64,6 +70,12 @@ impl Config {
             // https://github.com/rust-lang/docs.rs/pull/930#issuecomment-667729380
             max_parse_memory: env("DOCSRS_MAX_PARSE_MEMORY", 5 * 1024 * 1024)?,
             registry_gc_interval: env("DOCSRS_REGISTRY_GC_INTERVAL", 60 * 60)?,
+
+            rustwide_workspace: env("CRATESFYI_RUSTWIDE_WORKSPACE", PathBuf::from(".workspace"))?,
+            inside_docker: env("DOCS_RS_DOCKER", false)?,
+            local_docker_image: maybe_env("DOCS_RS_LOCAL_DOCKER_IMAGE")?,
+            toolchain: env("CRATESFYI_TOOLCHAIN", "nightly".to_string())?,
+            build_cpu_limit: maybe_env("DOCS_RS_BUILD_CPU_LIMIT")?,
         })
     }
 

src/context.rs

@@ -1,5 +1,5 @@
 use crate::db::Pool;
-use crate::{BuildQueue, Config, Metrics, Storage};
+use crate::{BuildQueue, Config, Index, Metrics, Storage};
 use failure::Error;
 use std::sync::Arc;
 
@@ -9,4 +9,5 @@ pub trait Context {
     fn storage(&self) -> Result<Arc<Storage>, Error>;
     fn pool(&self) -> Result<Pool, Error>;
     fn metrics(&self) -> Result<Arc<Metrics>, Error>;
+    fn index(&self) -> Result<Arc<Index>, Error>;
 }

src/docbuilder/mod.rs

@@ -1,6 +1,5 @@
 mod crates;
 mod limits;
-pub(crate) mod options;
 mod queue;
 mod rustwide_builder;
 
@@ -10,92 +9,29 @@ pub(crate) use self::rustwide_builder::{BuildResult, DocCoverage};
 
 use crate::db::Pool;
 use crate::error::Result;
-use crate::index::Index;
-use crate::BuildQueue;
-use crate::DocBuilderOptions;
-use log::debug;
-use std::collections::BTreeSet;
+use crate::{BuildQueue, Config};
 use std::fs;
-use std::io::prelude::*;
-use std::io::BufReader;
 use std::path::PathBuf;
 use std::sync::Arc;
 
 /// chroot based documentation builder
 pub struct DocBuilder {
-    options: DocBuilderOptions,
-    index: Index,
+    config: Arc<Config>,
     db: Pool,
     build_queue: Arc<BuildQueue>,
-    cache: BTreeSet<String>,
-    db_cache: BTreeSet<String>,
 }
 
 impl DocBuilder {
-    pub fn new(options: DocBuilderOptions, db: Pool, build_queue: Arc<BuildQueue>) -> DocBuilder {
-        let index = Index::new(&options.registry_index_path).expect("valid index");
+    pub fn new(config: Arc<Config>, db: Pool, build_queue: Arc<BuildQueue>) -> DocBuilder {
         DocBuilder {
+            config,
             build_queue,
-            options,
-            index,
             db,
-            cache: BTreeSet::new(),
-            db_cache: BTreeSet::new(),
         }
     }
 
-    /// Loads build cache
-    pub fn load_cache(&mut self) -> Result<()> {
-        debug!("Loading cache");
-
-        let path = PathBuf::from(&self.options.prefix).join("cache");
-        let reader = fs::File::open(path).map(BufReader::new);
-
-        if let Ok(reader) = reader {
-            for line in reader.lines() {
-                self.cache.insert(line?);
-            }
-        }
-
-        self.load_database_cache()?;
-
-        Ok(())
-    }
-
-    fn load_database_cache(&mut self) -> Result<()> {
-        debug!("Loading database cache");
-
-        let mut conn = self.db.get()?;
-        for row in &mut conn.query(
-            "SELECT name, version FROM crates, releases \
-             WHERE crates.id = releases.crate_id",
-            &[],
-        )? {
-            let name: String = row.get(0);
-            let version: String = row.get(1);
-
-            self.db_cache.insert(format!("{}-{}", name, version));
-        }
-
-        Ok(())
-    }
-
-    /// Saves build cache
-    pub fn save_cache(&self) -> Result<()> {
-        debug!("Saving cache");
-
-        let path = PathBuf::from(&self.options.prefix).join("cache");
-        let mut file = fs::OpenOptions::new().write(true).create(true).open(path)?;
-
-        for krate in &self.cache {
-            writeln!(file, "{}", krate)?;
-        }
-
-        Ok(())
-    }
-
     fn lock_path(&self) -> PathBuf {
-        self.options.prefix.join("cratesfyi.lock")
+        self.config.prefix.join("cratesfyi.lock")
     }
 
     /// Creates a lock file. Daemon will check this lock file and stop operating if its exists.
@@ -122,21 +58,4 @@ impl DocBuilder {
     pub fn is_locked(&self) -> bool {
         self.lock_path().exists()
     }
-
-    /// Returns a reference of options
-    pub fn options(&self) -> &DocBuilderOptions {
-        &self.options
-    }
-
-    fn add_to_cache(&mut self, name: &str, version: &str) {
-        self.cache.insert(format!("{}-{}", name, version));
-    }
-
-    fn should_build(&self, name: &str, version: &str) -> bool {
-        let name = format!("{}-{}", name, version);
-        let local = self.options.skip_if_log_exists && self.cache.contains(&name);
-        let db = self.options.skip_if_exists && self.db_cache.contains(&name);
-
-        !(local || db)
-    }
 }