Removed the content & versions column, removed the search updater service and semi-refactored CrateDetails::new

Kixiron · Joshua Nelson · commit d4089ba35833 · 2020-06-02T21:22:39.000-04:00
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
@@ -27,7 +27,6 @@ while true; do
 done
 set -e
 
-cratesfyi database update-search-index
 cratesfyi database update-release-activity
 
 if ! [ -d "${CRATESFYI_PREFIX}/crates.io-index/.git" ]; then
diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs
@@ -381,9 +381,6 @@ enum DatabaseSubcommand {
     /// Updates monthly release activity chart
     UpdateReleaseActivity,
 
-    /// Updates search index
-    UpdateSearchIndex,
-
     /// Removes a whole crate from the database
     DeleteCrate {
         /// Name of the crate to delete
@@ -420,11 +417,6 @@ impl DatabaseSubcommand {
             Self::UpdateReleaseActivity => cratesfyi::utils::update_release_activity()
                 .expect("Failed to update release activity"),
 
-            Self::UpdateSearchIndex => {
-                let conn = db::connect_db().expect("failed to connect to the database");
-                db::update_search_index(&conn).expect("Failed to update search index");
-            }
-
             Self::DeleteCrate { crate_name } => {
                 let conn = db::connect_db().expect("failed to connect to the database");
                 db::delete_crate(&conn, &crate_name).expect("failed to delete the crate");
diff --git a/src/db/add_package.rs b/src/db/add_package.rs
@@ -13,7 +13,6 @@ use crate::{
 use log::debug;
 use postgres::Connection;
 use regex::Regex;
-use semver::Version;
 use serde_json::Value;
 use slug::slugify;
 
@@ -118,31 +117,13 @@ pub(crate) fn add_package_into_database(
     add_authors_into_database(&conn, &metadata_pkg, release_id)?;
     add_owners_into_database(&conn, &cratesio_data.owners, crate_id)?;
 
-    // Update versions
-    {
-        let metadata_version = Version::parse(&metadata_pkg.version)?;
-        let mut versions: Value = conn
-            .query("SELECT versions FROM crates WHERE id = $1", &[&crate_id])?
-            .get(0)
-            .get(0);
-        if let Some(versions_array) = versions.as_array_mut() {
-            let mut found = false;
-            for version in versions_array.clone() {
-                let version = Version::parse(version.as_str().unwrap())?;
-                if version == metadata_version {
-                    found = true;
-                }
-            }
-
-            if !found {
-                versions_array.push(Value::String(metadata_pkg.version.clone()));
-            }
-        }
-        let _ = conn.query(
-            "UPDATE crates SET versions = $1 WHERE id = $2",
-            &[&versions, &crate_id],
-        );
-    }
+    // Update the crates table with the new release
+    conn.execute(
+        "UPDATE crates
+         SET latest_version_id = $2
+         WHERE id = $1",
+        &[&crate_id, &release_id],
+    )?;
 
     Ok(release_id)
 }
diff --git a/src/db/migrate.rs b/src/db/migrate.rs
@@ -325,6 +325,21 @@ pub fn migrate(version: Option<Version>, conn: &Connection) -> CratesfyiResult<(
                 ALTER TABLE releases ALTER COLUMN doc_targets DROP NOT NULL;
             "
         ),
+        migration!(
+            context,
+            // version
+            13,
+            // description
+            "Remove the content column and releases column",
+            // upgrade query
+            "ALTER TABLE crates
+             DROP COLUMN content,
+             DROP COLUMN versions;",
+            // downgrade query
+            "ALTER TABLE crates
+             ADD COLUMN content,
+             ADD COLUMN versions;"
+        ),
     ];
 
     for migration in migrations {
diff --git a/src/db/mod.rs b/src/db/mod.rs
@@ -7,7 +7,6 @@ pub use self::file::add_path_into_database;
 pub use self::migrate::migrate;
 
 use failure::Fail;
-use postgres::error::Error;
 use postgres::{Connection, TlsMode};
 use std::env;
 
@@ -54,42 +53,6 @@ pub(crate) fn create_pool() -> r2d2::Pool<r2d2_postgres::PostgresConnectionManag
         .expect("Failed to create r2d2 pool")
 }
 
-/// Updates content column in crates table.
-///
-/// This column will be used for searches and always contains `tsvector` of:
-///
-///   * crate name (rank A-weight)
-///   * latest release description (rank B-weight)
-///   * latest release keywords (rank B-weight)
-///   * latest release readme (rank C-weight)
-///   * latest release root rustdoc (rank C-weight)
-pub fn update_search_index(conn: &Connection) -> Result<u64, Error> {
-    conn.execute(
-        "
-        WITH doc as (
-            SELECT DISTINCT ON(releases.crate_id)
-                   releases.id,
-                   releases.crate_id,
-                   setweight(to_tsvector(crates.name), 'A')                                   ||
-                   setweight(to_tsvector(coalesce(releases.description, '')), 'B')            ||
-                   setweight(to_tsvector(coalesce((
-                                SELECT string_agg(value, ' ')
-                                FROM json_array_elements_text(releases.keywords)), '')), 'B')
-                    as content
-            FROM releases
-            INNER JOIN crates ON crates.id = releases.crate_id
-            ORDER BY releases.crate_id, releases.release_time DESC
-        )
-        UPDATE crates
-        SET latest_version_id = doc.id,
-            content = doc.content
-        FROM doc
-        WHERE crates.id = doc.crate_id AND
-            (crates.latest_version_id = 0 OR crates.latest_version_id != doc.id);",
-        &[],
-    )
-}
-
 #[cfg(test)]
 mod test {
     use super::*;
diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs
@@ -3,7 +3,6 @@
 //! This daemon will start web server, track new packages and build them
 
 use crate::{
-    db::{connect_db, update_search_index},
     docbuilder::RustwideBuilder,
     utils::{github_updater, pubsubhubbub, update_release_activity},
     DocBuilder, DocBuilderOptions,
@@ -232,18 +231,6 @@ pub fn start_daemon(background: bool) {
         })
         .unwrap();
 
-    // update search index every 3 hours
-    thread::Builder::new()
-        .name("search index updater".to_string())
-        .spawn(move || loop {
-            thread::sleep(Duration::from_secs(60 * 60 * 3));
-            let conn = connect_db().expect("Failed to connect database");
-            if let Err(e) = update_search_index(&conn) {
-                error!("Failed to update search index: {}", e);
-            }
-        })
-        .unwrap();
-
     // update github stats every 6 hours
     thread::Builder::new()
         .name("github stat updater".to_string())
diff --git a/src/web/crate_details.rs b/src/web/crate_details.rs
@@ -115,8 +115,10 @@ pub struct Release {
 impl CrateDetails {
     pub fn new(conn: &Connection, name: &str, version: &str) -> Option<CrateDetails> {
         // get all stuff, I love you rustfmt
-        let query = "SELECT crates.id,
-                releases.id,
+        let query = "
+            SELECT
+                crates.id AS crate_id,
+                releases.id AS release_id,
                 crates.name,
                 releases.version,
                 releases.description,
@@ -132,7 +134,7 @@ impl CrateDetails {
                 releases.keywords,
                 releases.have_examples,
                 releases.target_name,
-                crates.versions,
+                ARRAY(SELECT releases.version FROM releases WHERE releases.crate_id = crates.id) AS versions,
                 crates.github_stars,
                 crates.github_forks,
                 crates.github_issues,
@@ -142,59 +144,48 @@ impl CrateDetails {
                 releases.license,
                 releases.documentation_url,
                 releases.default_target
-         FROM releases
-         INNER JOIN crates ON releases.crate_id = crates.id
-         WHERE crates.name = $1 AND releases.version = $2;";
+            FROM releases
+            INNER JOIN crates ON releases.crate_id = crates.id
+            WHERE crates.name = $1 AND releases.version = $2;";
 
         let rows = conn.query(query, &[&name, &version]).unwrap();
 
-        if rows.is_empty() {
+        let krate = if rows.is_empty() {
             return None;
-        }
+        } else {
+            rows.get(0)
+        };
 
-        let crate_id: i32 = rows.get(0).get(0);
-        let release_id: i32 = rows.get(0).get(1);
+        let crate_id: i32 = krate.get("crate_id");
+        let release_id: i32 = krate.get("release_id");
 
         // sort versions with semver
         let releases = {
-            let versions_from_db: Value = rows.get(0).get(17);
-
-            if let Some(versions_from_db) = versions_from_db.as_array() {
-                let mut versions: Vec<semver::Version> = versions_from_db
-                    .iter()
-                    .filter_map(|version| {
-                        if let Some(version) = version.as_str() {
-                            if let Ok(sem_ver) = semver::Version::parse(&version) {
-                                return Some(sem_ver);
-                            }
-                        }
-
-                        None
-                    })
-                    .collect();
-
-                versions.sort();
-                versions.reverse();
-                versions
-                    .iter()
-                    .map(|version| map_to_release(&conn, crate_id, version.to_string()))
-                    .collect()
-            } else {
-                Vec::new()
-            }
+            let versions: Vec<String> = krate.get("versions");
+            let mut versions: Vec<semver::Version> = versions
+                .iter()
+                .filter_map(|version| semver::Version::parse(&version).ok())
+                .collect();
+
+            versions.sort();
+            versions.reverse();
+            versions
+                .iter()
+                .map(|version| map_to_release(&conn, crate_id, version.to_string()))
+                .collect()
         };
 
         let metadata = MetaData {
-            name: rows.get(0).get(2),
-            version: rows.get(0).get(3),
-            description: rows.get(0).get(4),
-            rustdoc_status: rows.get(0).get(11),
-            target_name: rows.get(0).get(16),
-            default_target: rows.get(0).get(26),
+            name: krate.get("name"),
+            version: krate.get("version"),
+            description: krate.get("description"),
+            rustdoc_status: krate.get("rustdoc_status"),
+            target_name: krate.get("target_name"),
+            default_target: krate.get("default_target"),
         };
 
         let doc_targets = {
-            let data: Value = rows.get(0).get(23);
+            let data: Value = krate.get("doc_targets");
             data.as_array()
                 .map(|array| {
                     array
@@ -206,35 +197,35 @@ impl CrateDetails {
         };
 
         let mut crate_details = CrateDetails {
-            name: rows.get(0).get(2),
-            version: rows.get(0).get(3),
-            description: rows.get(0).get(4),
+            name: krate.get("name"),
+            version: krate.get("version"),
+            description: krate.get("description"),
             authors: Vec::new(),
             owners: Vec::new(),
-            authors_json: rows.get(0).get(5),
-            dependencies: rows.get(0).get(6),
-            readme: rows.get(0).get(7),
-            rustdoc: rows.get(0).get(8),
-            release_time: rows.get(0).get(9),
-            build_status: rows.get(0).get(10),
+            authors_json: krate.get("authors"),
+            dependencies: krate.get("dependencies"),
+            readme: krate.get("readme"),
+            rustdoc: krate.get("description_long"),
+            release_time: krate.get("release_time"),
+            build_status: krate.get("build_status"),
             last_successful_build: None,
-            rustdoc_status: rows.get(0).get(11),
-            repository_url: rows.get(0).get(12),
-            homepage_url: rows.get(0).get(13),
-            keywords: rows.get(0).get(14),
-            have_examples: rows.get(0).get(15),
-            target_name: rows.get(0).get(16),
+            rustdoc_status: krate.get("rustdoc_status"),
+            repository_url: krate.get("repository_url"),
+            homepage_url: krate.get("homepage_url"),
+            keywords: krate.get("keywords"),
+            have_examples: krate.get("have_examples"),
+            target_name: krate.get("target_name"),
             releases,
             github: false,
-            github_stars: rows.get(0).get(18),
-            github_forks: rows.get(0).get(19),
-            github_issues: rows.get(0).get(20),
+            github_stars: krate.get("github_stars"),
+            github_forks: krate.get("github_forks"),
+            github_issues: krate.get("github_issues"),
             metadata,
-            is_library: rows.get(0).get(21),
-            yanked: rows.get(0).get(22),
+            is_library: krate.get("is_library"),
+            yanked: krate.get("yanked"),
             doc_targets,
-            license: rows.get(0).get(24),
-            documentation_url: rows.get(0).get(25),
+            license: krate.get("license"),
+            documentation_url: krate.get("documentation_url"),
         };
 
         if let Some(repository_url) = crate_details.repository_url.clone() {
@@ -255,7 +246,7 @@ impl CrateDetails {
 
         crate_details.authors = authors
             .into_iter()
-            .map(|row| (row.get(0), row.get(1)))
+            .map(|row| (row.get("name"), row.get("slug")))
             .collect();
 
         // get owners
@@ -271,7 +262,7 @@ impl CrateDetails {
 
         crate_details.owners = owners
             .into_iter()
-            .map(|row| (row.get(0), row.get(1)))
+            .map(|row| (row.get("login"), row.get("avatar")))
             .collect();
 
         if !crate_details.build_status {
