Skip to content

[Refactor] Removed the search index updater #804

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 3, 2020
Merged

[Refactor] Removed the search index updater #804

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion docker-entrypoint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ while true; do
done
set -e

cratesfyi database update-search-index
cratesfyi database update-release-activity

if ! [ -d "${CRATESFYI_PREFIX}/crates.io-index/.git" ]; then
Expand Down
8 changes: 0 additions & 8 deletions src/bin/cratesfyi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,9 +381,6 @@ enum DatabaseSubcommand {
/// Updates monthly release activity chart
UpdateReleaseActivity,

/// Updates search index
UpdateSearchIndex,

/// Removes a whole crate from the database
DeleteCrate {
/// Name of the crate to delete
Expand Down Expand Up @@ -420,11 +417,6 @@ impl DatabaseSubcommand {
Self::UpdateReleaseActivity => cratesfyi::utils::update_release_activity()
.expect("Failed to update release activity"),

Self::UpdateSearchIndex => {
let conn = db::connect_db().expect("failed to connect to the database");
db::update_search_index(&conn).expect("Failed to update search index");
}

Self::DeleteCrate { crate_name } => {
let conn = db::connect_db().expect("failed to connect to the database");
db::delete_crate(&conn, &crate_name).expect("failed to delete the crate");
Expand Down
33 changes: 7 additions & 26 deletions src/db/add_package.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ use crate::{
use log::debug;
use postgres::Connection;
use regex::Regex;
use semver::Version;
use serde_json::Value;
use slug::slugify;

Expand Down Expand Up @@ -118,31 +117,13 @@ pub(crate) fn add_package_into_database(
add_authors_into_database(&conn, &metadata_pkg, release_id)?;
add_owners_into_database(&conn, &cratesio_data.owners, crate_id)?;

// Update versions
{
let metadata_version = Version::parse(&metadata_pkg.version)?;
let mut versions: Value = conn
.query("SELECT versions FROM crates WHERE id = $1", &[&crate_id])?
.get(0)
.get(0);
if let Some(versions_array) = versions.as_array_mut() {
let mut found = false;
for version in versions_array.clone() {
let version = Version::parse(version.as_str().unwrap())?;
if version == metadata_version {
found = true;
}
}

if !found {
versions_array.push(Value::String(metadata_pkg.version.clone()));
}
}
let _ = conn.query(
"UPDATE crates SET versions = $1 WHERE id = $2",
&[&versions, &crate_id],
);
}
// Update the crates table with the new release
conn.execute(
"UPDATE crates
SET latest_version_id = $2
WHERE id = $1",
&[&crate_id, &release_id],
)?;

Ok(release_id)
}
Expand Down
15 changes: 15 additions & 0 deletions src/db/migrate.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,21 @@ pub fn migrate(version: Option<Version>, conn: &Connection) -> CratesfyiResult<(
ALTER TABLE releases ALTER COLUMN doc_targets DROP NOT NULL;
"
),
migration!(
context,
// version
13,
// description
"Remove the content column and releases column",
// upgrade query
"ALTER TABLE crates
DROP COLUMN content,
DROP COLUMN versions;",
// downgrade query
"ALTER TABLE crates
ADD COLUMN content,
ADD COLUMN versions;"
),
];

for migration in migrations {
Expand Down
37 changes: 0 additions & 37 deletions src/db/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ pub use self::file::add_path_into_database;
pub use self::migrate::migrate;

use failure::Fail;
use postgres::error::Error;
use postgres::{Connection, TlsMode};
use std::env;

Expand Down Expand Up @@ -54,42 +53,6 @@ pub(crate) fn create_pool() -> r2d2::Pool<r2d2_postgres::PostgresConnectionManag
.expect("Failed to create r2d2 pool")
}

/// Updates content column in crates table.
///
/// This column will be used for searches and always contains `tsvector` of:
///
/// * crate name (rank A-weight)
/// * latest release description (rank B-weight)
/// * latest release keywords (rank B-weight)
/// * latest release readme (rank C-weight)
/// * latest release root rustdoc (rank C-weight)
pub fn update_search_index(conn: &Connection) -> Result<u64, Error> {
conn.execute(
"
WITH doc as (
SELECT DISTINCT ON(releases.crate_id)
releases.id,
releases.crate_id,
setweight(to_tsvector(crates.name), 'A') ||
setweight(to_tsvector(coalesce(releases.description, '')), 'B') ||
setweight(to_tsvector(coalesce((
SELECT string_agg(value, ' ')
FROM json_array_elements_text(releases.keywords)), '')), 'B')
as content
FROM releases
INNER JOIN crates ON crates.id = releases.crate_id
ORDER BY releases.crate_id, releases.release_time DESC
)
UPDATE crates
SET latest_version_id = doc.id,
content = doc.content
FROM doc
WHERE crates.id = doc.crate_id AND
(crates.latest_version_id = 0 OR crates.latest_version_id != doc.id);",
&[],
)
}

#[cfg(test)]
mod test {
use super::*;
Expand Down
13 changes: 0 additions & 13 deletions src/utils/daemon.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
//! This daemon will start web server, track new packages and build them

use crate::{
db::{connect_db, update_search_index},
docbuilder::RustwideBuilder,
utils::{github_updater, pubsubhubbub, update_release_activity},
DocBuilder, DocBuilderOptions,
Expand Down Expand Up @@ -232,18 +231,6 @@ pub fn start_daemon(background: bool) {
})
.unwrap();

// update search index every 3 hours
thread::Builder::new()
.name("search index updater".to_string())
.spawn(move || loop {
thread::sleep(Duration::from_secs(60 * 60 * 3));
let conn = connect_db().expect("Failed to connect database");
if let Err(e) = update_search_index(&conn) {
error!("Failed to update search index: {}", e);
}
})
.unwrap();

// update github stats every 6 hours
thread::Builder::new()
.name("github stat updater".to_string())
Expand Down
123 changes: 57 additions & 66 deletions src/web/crate_details.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,10 @@ pub struct Release {
impl CrateDetails {
pub fn new(conn: &Connection, name: &str, version: &str) -> Option<CrateDetails> {
// get all stuff, I love you rustfmt
let query = "SELECT crates.id,
releases.id,
let query = "
SELECT
crates.id AS crate_id,
releases.id AS release_id,
crates.name,
releases.version,
releases.description,
Expand All @@ -132,7 +134,7 @@ impl CrateDetails {
releases.keywords,
releases.have_examples,
releases.target_name,
crates.versions,
ARRAY(SELECT releases.version FROM releases WHERE releases.crate_id = crates.id) AS versions,
crates.github_stars,
crates.github_forks,
crates.github_issues,
Expand All @@ -142,59 +144,48 @@ impl CrateDetails {
releases.license,
releases.documentation_url,
releases.default_target
FROM releases
INNER JOIN crates ON releases.crate_id = crates.id
WHERE crates.name = $1 AND releases.version = $2;";
FROM releases
INNER JOIN crates ON releases.crate_id = crates.id
WHERE crates.name = $1 AND releases.version = $2;";

let rows = conn.query(query, &[&name, &version]).unwrap();

if rows.is_empty() {
let krate = if rows.is_empty() {
return None;
}
} else {
rows.get(0)
};

let crate_id: i32 = rows.get(0).get(0);
let release_id: i32 = rows.get(0).get(1);
let crate_id: i32 = krate.get("crate_id");
let release_id: i32 = krate.get("release_id");

// sort versions with semver
let releases = {
let versions_from_db: Value = rows.get(0).get(17);

if let Some(versions_from_db) = versions_from_db.as_array() {
let mut versions: Vec<semver::Version> = versions_from_db
.iter()
.filter_map(|version| {
if let Some(version) = version.as_str() {
if let Ok(sem_ver) = semver::Version::parse(&version) {
return Some(sem_ver);
}
}

None
})
.collect();

versions.sort();
versions.reverse();
versions
.iter()
.map(|version| map_to_release(&conn, crate_id, version.to_string()))
.collect()
} else {
Vec::new()
}
let versions: Vec<String> = krate.get("versions");
let mut versions: Vec<semver::Version> = versions
.iter()
.filter_map(|version| semver::Version::parse(&version).ok())
.collect();

versions.sort();
versions.reverse();
versions
.iter()
.map(|version| map_to_release(&conn, crate_id, version.to_string()))
.collect()
};

let metadata = MetaData {
name: rows.get(0).get(2),
version: rows.get(0).get(3),
description: rows.get(0).get(4),
rustdoc_status: rows.get(0).get(11),
target_name: rows.get(0).get(16),
default_target: rows.get(0).get(26),
name: krate.get("name"),
version: krate.get("version"),
description: krate.get("description"),
rustdoc_status: krate.get("rustdoc_status"),
target_name: krate.get("target_name"),
default_target: krate.get("default_target"),
};

let doc_targets = {
let data: Value = rows.get(0).get(23);
let data: Value = krate.get("doc_targets");
data.as_array()
.map(|array| {
array
Expand All @@ -206,35 +197,35 @@ impl CrateDetails {
};

let mut crate_details = CrateDetails {
name: rows.get(0).get(2),
version: rows.get(0).get(3),
description: rows.get(0).get(4),
name: krate.get("name"),
version: krate.get("version"),
description: krate.get("description"),
authors: Vec::new(),
owners: Vec::new(),
authors_json: rows.get(0).get(5),
dependencies: rows.get(0).get(6),
readme: rows.get(0).get(7),
rustdoc: rows.get(0).get(8),
release_time: rows.get(0).get(9),
build_status: rows.get(0).get(10),
authors_json: krate.get("authors"),
dependencies: krate.get("dependencies"),
readme: krate.get("readme"),
rustdoc: krate.get("description_long"),
release_time: krate.get("release_time"),
build_status: krate.get("build_status"),
last_successful_build: None,
rustdoc_status: rows.get(0).get(11),
repository_url: rows.get(0).get(12),
homepage_url: rows.get(0).get(13),
keywords: rows.get(0).get(14),
have_examples: rows.get(0).get(15),
target_name: rows.get(0).get(16),
rustdoc_status: krate.get("rustdoc_status"),
repository_url: krate.get("repository_url"),
homepage_url: krate.get("homepage_url"),
keywords: krate.get("keywords"),
have_examples: krate.get("have_examples"),
target_name: krate.get("target_name"),
releases,
github: false,
github_stars: rows.get(0).get(18),
github_forks: rows.get(0).get(19),
github_issues: rows.get(0).get(20),
github_stars: krate.get("github_stars"),
github_forks: krate.get("github_forks"),
github_issues: krate.get("github_issues"),
metadata,
is_library: rows.get(0).get(21),
yanked: rows.get(0).get(22),
is_library: krate.get("is_library"),
yanked: krate.get("yanked"),
doc_targets,
license: rows.get(0).get(24),
documentation_url: rows.get(0).get(25),
license: krate.get("license"),
documentation_url: krate.get("documentation_url"),
};

if let Some(repository_url) = crate_details.repository_url.clone() {
Expand All @@ -255,7 +246,7 @@ impl CrateDetails {

crate_details.authors = authors
.into_iter()
.map(|row| (row.get(0), row.get(1)))
.map(|row| (row.get("name"), row.get("slug")))
.collect();

// get owners
Expand All @@ -271,7 +262,7 @@ impl CrateDetails {

crate_details.owners = owners
.into_iter()
.map(|row| (row.get(0), row.get(1)))
.map(|row| (row.get("login"), row.get("avatar")))
.collect();

if !crate_details.build_status {
Expand Down