Trivial consistency check

Cargo.toml

@@ -12,6 +12,7 @@ edition = "2018"
 log = "0.4"
 regex = "1"
 structopt = "0.3"
+crates-index = "0.15.1"
 crates-index-diff = "7"
 reqwest = { version = "0.10.6", features = ["blocking", "json"] } # TODO: Remove blocking when async is ready
 semver = { version = "0.9", features = ["serde"] }

src/bin/cratesfyi.rs

@@ -443,6 +443,13 @@ enum DatabaseSubcommand {
         #[structopt(subcommand)]
         command: BlacklistSubcommand,
     },
+
+    /// Compares the database with the index and resolves inconsistencies
+    Synchronize {
+        /// Don't actually resolve the inconsistencies, just log them
+        #[structopt(long)]
+        dry_run: bool,
+    },
 }
 
 impl DatabaseSubcommand {
@@ -488,6 +495,14 @@ impl DatabaseSubcommand {
             } => db::delete_crate(&mut *ctx.conn()?, &*ctx.storage()?, &name)
                 .context("failed to delete the crate")?,
             Self::Blacklist { command } => command.handle_args(ctx)?,
+
+            Self::Synchronize { dry_run } => {
+                cratesfyi::utils::consistency::run_check(
+                    &*ctx.config()?,
+                    &mut *ctx.conn()?,
+                    dry_run,
+                )?;
+            }
         }
         Ok(())
     }

src/index/crates.rs

@@ -0,0 +1,47 @@
+use crates_index::Crate;
+use failure::ResultExt;
+
+pub(crate) struct Crates {
+    repo: git2::Repository,
+}
+
+impl Crates {
+    pub(super) fn new(repo: git2::Repository) -> Self {
+        Self { repo }
+    }
+
+    pub(crate) fn walk(&self, mut f: impl FnMut(Crate)) -> Result<(), failure::Error> {
+        log::debug!("Walking crates in index");
+        let tree = self
+            .repo
+            .find_commit(self.repo.refname_to_id("refs/remotes/origin/master")?)?
+            .tree()?;
+
+        let mut result = Ok(());
+
+        tree.walk(git2::TreeWalkMode::PreOrder, |_, entry| {
+            result = (|| {
+                if let Some(blob) = entry.to_object(&self.repo)?.as_blob() {
+                    if let Ok(krate) = Crate::from_slice(blob.content()) {
+                        f(krate);
+                    } else {
+                        log::warn!("not a crate '{}'", entry.name().unwrap());
+                    }
+                }
+                Result::<(), failure::Error>::Ok(())
+            })()
+            .with_context(|_| {
+                format!(
+                    "loading crate details from '{}'",
+                    entry.name().unwrap_or("<unknown>")
+                )
+            });
+            match result {
+                Ok(_) => git2::TreeWalkResult::Ok,
+                Err(_) => git2::TreeWalkResult::Abort,
+            }
+        })?;
+
+        Ok(result?)
+    }
+}

src/index/mod.rs

@@ -2,11 +2,12 @@ use std::path::{Path, PathBuf};
 
 use url::Url;
 
-use self::api::Api;
+use self::{api::Api, crates::Crates};
 use crate::error::Result;
 use failure::ResultExt;
 
 pub(crate) mod api;
+mod crates;
 
 pub struct Index {
     path: PathBuf,
@@ -56,6 +57,17 @@ impl Index {
         Ok(diff)
     }
 
+    pub(crate) fn crates(&self) -> Result<Crates> {
+        // First ensure the index is up to date, peeking will pull the latest changes without
+        // affecting anything else.
+        log::debug!("Updating index");
+        self.diff()?.peek_changes()?;
+        // It'd be nice to use `crates_index` directly for interacting with the index, but it
+        // doesn't support bare repositories. So we use its `Crate` type but walk the index
+        // ourselves.
+        Ok(Crates::new(git2::Repository::open(&self.path)?))
+    }
+
     pub fn api(&self) -> &Api {
         &self.api
     }

src/utils/consistency/data.rs

@@ -0,0 +1,48 @@
+use std::{
+    cmp::PartialEq,
+    collections::BTreeMap,
+    fmt::{self, Debug, Display, Formatter},
+};
+
+#[derive(Default, Debug)]
+pub(crate) struct Data {
+    pub(crate) crates: BTreeMap<CrateName, Crate>,
+}
+
+#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Default, Debug)]
+pub(crate) struct CrateName(pub(crate) String);
+
+#[derive(Default, Debug)]
+pub(crate) struct Crate {
+    pub(crate) releases: BTreeMap<Version, Release>,
+}
+
+#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Default, Debug)]
+pub(crate) struct Version(pub(crate) String);
+
+#[derive(Default, Debug)]
+pub(crate) struct Release {}
+
+impl PartialEq<String> for CrateName {
+    fn eq(&self, other: &String) -> bool {
+        self.0 == *other
+    }
+}
+
+impl PartialEq<String> for Version {
+    fn eq(&self, other: &String) -> bool {
+        self.0 == *other
+    }
+}
+
+impl Display for CrateName {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        Display::fmt(&self.0, f)
+    }
+}
+
+impl Display for Version {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        Display::fmt(&self.0, f)
+    }
+}

src/utils/consistency/db.rs

@@ -0,0 +1,60 @@
+use super::data::{Crate, CrateName, Data, Release, Version};
+use std::collections::BTreeMap;
+
+pub(crate) fn load(conn: &mut postgres::Client) -> Result<Data, failure::Error> {
+    let rows = conn.query(
+        "
+        SELECT
+            crates.name,
+            releases.version
+        FROM crates
+        INNER JOIN releases ON releases.crate_id = crates.id
+        ORDER BY crates.id, releases.id
+    ",
+        &[],
+    )?;
+
+    let mut data = Data {
+        crates: BTreeMap::new(),
+    };
+
+    let mut rows = rows.iter();
+
+    struct Current {
+        name: CrateName,
+        krate: Crate,
+    }
+
+    let mut current = if let Some(row) = rows.next() {
+        Current {
+            name: CrateName(row.get("name")),
+            krate: Crate {
+                releases: {
+                    let mut releases = BTreeMap::new();
+                    releases.insert(Version(row.get("version")), Release {});
+                    releases
+                },
+            },
+        }
+    } else {
+        return Ok(data);
+    };
+
+    for row in rows {
+        let name = row.get("name");
+        if current.name != name {
+            data.crates.insert(
+                std::mem::replace(&mut current.name, CrateName(name)),
+                std::mem::take(&mut current.krate),
+            );
+        }
+        current
+            .krate
+            .releases
+            .insert(Version(row.get("version")), Release::default());
+    }
+
+    data.crates.insert(current.name, current.krate);
+
+    Ok(data)
+}