diff --git a/Cargo.lock b/Cargo.lock index bd4f34a9c2e..21190b19d8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,6 +35,7 @@ dependencies = [ "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "s3 0.0.1", "semver 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 0.9.9 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -266,6 +267,7 @@ dependencies = [ "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "pq-sys 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 0.9.9 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index dfb0f31029c..995a7ca034e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,10 +36,11 @@ rustc-serialize = "0.3" license-exprs = "^1.3" dotenv = "0.8.0" toml = "0.2" -diesel = { version = "0.11.0", features = ["postgres", "serde_json"] } +diesel = { version = "0.11.0", features = ["postgres", "serde_json", "deprecated-time"] } diesel_codegen = { version = "0.11.0", features = ["postgres"] } r2d2-diesel = "0.11.0" diesel_full_text_search = "0.11.0" +serde_json = "0.9.0" conduit = "0.8" conduit-conditional-get = "0.8" diff --git a/migrations/20170307211844_versions_yanked_is_not_nullalbe/down.sql b/migrations/20170307211844_versions_yanked_is_not_nullalbe/down.sql new file mode 100644 index 00000000000..9752b0e6186 --- /dev/null +++ b/migrations/20170307211844_versions_yanked_is_not_nullalbe/down.sql @@ -0,0 +1 @@ +ALTER TABLE versions ALTER COLUMN yanked DROP NOT NULL; diff --git a/migrations/20170307211844_versions_yanked_is_not_nullalbe/up.sql b/migrations/20170307211844_versions_yanked_is_not_nullalbe/up.sql new file mode 100644 index 00000000000..9c636b1e818 --- /dev/null +++ b/migrations/20170307211844_versions_yanked_is_not_nullalbe/up.sql @@ -0,0 +1 @@ +ALTER TABLE versions ALTER COLUMN yanked SET NOT NULL; diff --git a/src/app.rs b/src/app.rs index 764908ae944..3d3851d2b91 100644 --- a/src/app.rs +++ b/src/app.rs @@ -54,7 +54,8 @@ impl App { .helper_threads(if config.env == ::Env::Production {3} else {1}) .build(); let diesel_db_config = r2d2::Config::builder() - .pool_size(if config.env == ::Env::Production {1} else {1}) + .pool_size(if config.env == ::Env::Production {50} else {1}) + .min_idle(if config.env == ::Env::Production {5} else {1}) .helper_threads(if config.env == ::Env::Production {3} else {1}) .build(); diff --git a/src/badge.rs b/src/badge.rs index 30dea5a8bee..8e4dee4fc89 100644 --- a/src/badge.rs +++ b/src/badge.rs @@ -1,11 +1,15 @@ -use util::CargoResult; -use krate::Crate; use Model; +use krate::Crate; +use schema::badges; +use util::CargoResult; -use std::collections::HashMap; +use diesel::prelude::*; +use diesel::pg::Pg; use pg::GenericConnection; use pg::rows::Row; use rustc_serialize::json::Json; +use serde_json; +use std::collections::HashMap; #[derive(Debug, PartialEq, Clone)] pub enum Badge { @@ -26,6 +30,17 @@ pub struct EncodableBadge { pub attributes: HashMap, } +impl Queryable for Badge { + type Row = (i32, String, serde_json::Value); + + fn build((_, badge_type, attributes): Self::Row) -> Self { + let attributes = serde_json::from_value::>(attributes) + .expect("attributes was not a map in the database"); + Self::from_attributes(&badge_type, &attributes) + .expect("invalid badge in the database") + } +} + impl Model for Badge { fn from_row(row: &Row) -> Badge { let attributes: Json = row.get("attributes"); diff --git a/src/category.rs b/src/category.rs index f209acea690..8e06696330c 100644 --- a/src/category.rs +++ b/src/category.rs @@ -8,10 +8,13 @@ use pg::rows::Row; use {Model, Crate}; use db::RequestTransaction; +use schema::*; use util::{RequestUtils, CargoResult, ChainError}; use util::errors::NotFound; -#[derive(Clone)] +#[derive(Clone, Identifiable, Associations)] +#[has_many(crates_categories)] +#[table_name="categories"] pub struct Category { pub id: i32, pub category: String, @@ -21,6 +24,15 @@ pub struct Category { pub crates_cnt: i32, } +#[derive(Associations)] +#[belongs_to(Category)] +#[table_name="crates_categories"] +#[allow(dead_code)] // FIXME: Hey @sgrif add better many-to-many support to Diesel, wouldya? +struct CrateCategory { + crate_id: i32, + category_id: i32, +} + #[derive(RustcEncodable, RustcDecodable)] pub struct EncodableCategory { pub id: String, diff --git a/src/keyword.rs b/src/keyword.rs index 6ebc5b64ab2..1d3f7346e75 100644 --- a/src/keyword.rs +++ b/src/keyword.rs @@ -9,10 +9,12 @@ use pg::rows::Row; use {Model, Crate}; use db::RequestTransaction; +use schema::*; use util::{RequestUtils, CargoResult, ChainError, internal}; use util::errors::NotFound; -#[derive(Clone)] +#[derive(Clone, Identifiable, Associations)] +#[has_many(crates_keywords)] pub struct Keyword { pub id: i32, pub keyword: String, @@ -20,6 +22,15 @@ pub struct Keyword { pub crates_cnt: i32, } +#[derive(Associations)] +#[belongs_to(Keyword)] +#[table_name="crates_keywords"] +#[allow(dead_code)] +struct CrateKeyword { + crate_id: i32, + keyword_id: i32, +} + #[derive(RustcEncodable, RustcDecodable)] pub struct EncodableKeyword { pub id: String, diff --git a/src/krate.rs b/src/krate.rs index d4f4afb389a..0b839d17ca3 100644 --- a/src/krate.rs +++ b/src/krate.rs @@ -9,10 +9,12 @@ use std::sync::Arc; use conduit::{Request, Response}; use conduit_router::RequestParams; use curl::easy::Easy; +use diesel::prelude::*; +use diesel::pg::Pg; +use diesel_full_text_search::*; use license_exprs; use pg::GenericConnection; use pg::rows::Row; -use pg::types::ToSql; use pg; use rustc_serialize::hex::ToHex; use rustc_serialize::json; @@ -36,8 +38,9 @@ use util::errors::NotFound; use util::{LimitErrorReader, HashingReader}; use util::{RequestUtils, CargoResult, internal, ChainError, human}; use version::EncodableVersion; +use schema::*; -#[derive(Clone)] +#[derive(Clone, Queryable, Identifiable)] pub struct Crate { pub id: i32, pub name: String, @@ -53,6 +56,19 @@ pub struct Crate { pub max_upload_size: Option, } +/// We literally never want to select textsearchable_index_col +/// so we provide this type and constant to pass to `.select` +type AllColumns = (crates::id, crates::name, crates::updated_at, + crates::created_at, crates::downloads, crates::description, + crates::homepage, crates::documentation, crates::readme, crates::license, + crates::repository, crates::max_upload_size); + +pub const ALL_COLUMNS: AllColumns = (crates::id, crates::name, + crates::updated_at, crates::created_at, crates::downloads, + crates::description, crates::homepage, crates::documentation, + crates::readme, crates::license, crates::repository, + crates::max_upload_size); + #[derive(RustcEncodable, RustcDecodable)] pub struct EncodableCrate { pub id: String, @@ -486,136 +502,84 @@ impl Model for Crate { /// Handles the `GET /crates` route. #[allow(trivial_casts)] pub fn index(req: &mut Request) -> CargoResult { - let conn = req.tx()?; + let conn = req.db_conn()?; let (offset, limit) = req.pagination(10, 100)?; - let query = req.query(); - let sort = query.get("sort").map(|s| &s[..]).unwrap_or("alpha"); - let sort_sql = match sort { - "downloads" => "crates.downloads DESC", - _ => "crates.name ASC", - }; - - // Different queries for different parameters. - // - // Sure wish we had an arel-like thing here... - let mut pattern = String::new(); - let mut id = -1; - let (mut needs_id, mut needs_pattern) = (false, false); - let mut args = vec![&limit as &ToSql, &offset]; - let (q, cnt) = query.get("q").map(|query| { - args.insert(0, query); - let rank_sort_sql = match sort { - "downloads" => format!("{}, rank DESC", sort_sql), - _ => format!("rank DESC, {}", sort_sql), - }; - (format!("SELECT crates.* FROM crates, - plainto_tsquery($1) q, - ts_rank_cd(textsearchable_index_col, q) rank - WHERE q @@ textsearchable_index_col - ORDER BY name = $1 DESC, {} - LIMIT $2 OFFSET $3", rank_sort_sql), - "SELECT COUNT(crates.*) FROM crates, - plainto_tsquery($1) q - WHERE q @@ textsearchable_index_col".to_string()) - }).or_else(|| { - query.get("letter").map(|letter| { - pattern = format!("{}%", letter.chars().next().unwrap() + let params = req.query(); + + // This is a bit of a hack, but Boxed queries in Diesel don't implement `.clone()` and we need + // the query twice so we can `.count` it. This function is basically free, but it'd be nice to + // not have to wrap this in a funciton. + fn crates_query<'a>(params: &'a HashMap, user: CargoResult<&User>) + -> CargoResult::SqlType>> { + let mut query = crates::table.select(ALL_COLUMNS).into_boxed(); + + if let Some(q_string) = params.get("q") { + let q = plainto_tsquery(q_string); + query = query.filter(q.matches(crates::textsearchable_index_col)); + } else if let Some(letter) = params.get("letter") { + let pattern = format!("{}%", letter.chars().next().unwrap() .to_lowercase().collect::()); - needs_pattern = true; - (format!("SELECT * FROM crates WHERE canon_crate_name(name) \ - LIKE $1 ORDER BY {} LIMIT $2 OFFSET $3", sort_sql), - "SELECT COUNT(*) FROM crates WHERE canon_crate_name(name) \ - LIKE $1".to_string()) - }) - }).or_else(|| { - query.get("keyword").map(|kw| { - args.insert(0, kw); - let base = "FROM crates - INNER JOIN crates_keywords - ON crates.id = crates_keywords.crate_id - INNER JOIN keywords - ON crates_keywords.keyword_id = keywords.id - WHERE lower(keywords.keyword) = lower($1)"; - (format!("SELECT crates.* {} ORDER BY {} LIMIT $2 OFFSET $3", base, sort_sql), - format!("SELECT COUNT(crates.*) {}", base)) - }) - }).or_else(|| { - query.get("category").map(|cat| { - args.insert(0, cat); - let base = "FROM crates - INNER JOIN crates_categories - ON crates.id = crates_categories.crate_id - INNER JOIN categories - ON crates_categories.category_id = - categories.id - WHERE categories.slug = $1 OR - categories.slug LIKE $1 || '::%'"; - (format!("SELECT DISTINCT crates.* {} ORDER BY {} LIMIT $2 OFFSET $3", base, sort_sql), - format!("SELECT COUNT(DISTINCT crates.*) {}", base)) - }) - }).or_else(|| { - query.get("user_id").and_then(|s| s.parse::().ok()).map(|user_id| { - id = user_id; - needs_id = true; - (format!("SELECT crates.* FROM crates - INNER JOIN crate_owners - ON crate_owners.crate_id = crates.id - WHERE crate_owners.owner_id = $1 - AND crate_owners.owner_kind = {} - ORDER BY {} - LIMIT $2 OFFSET $3", - OwnerKind::User as i32, sort_sql), - format!("SELECT COUNT(crates.*) FROM crates - INNER JOIN crate_owners - ON crate_owners.crate_id = crates.id - WHERE crate_owners.owner_id = $1 \ - AND crate_owners.owner_kind = {}", - OwnerKind::User as i32)) - }) - }).or_else(|| { - query.get("following").map(|_| { - needs_id = true; - (format!("SELECT crates.* FROM crates - INNER JOIN follows - ON follows.crate_id = crates.id AND - follows.user_id = $1 ORDER BY - {} LIMIT $2 OFFSET $3", sort_sql), - "SELECT COUNT(crates.*) FROM crates - INNER JOIN follows - ON follows.crate_id = crates.id AND - follows.user_id = $1".to_string()) - }) - }).unwrap_or_else(|| { - (format!("SELECT * FROM crates ORDER BY {} LIMIT $1 OFFSET $2", - sort_sql), - "SELECT COUNT(*) FROM crates".to_string()) - }); - - if needs_id { - if id == -1 { - id = req.user()?.id; + query = query.filter(canon_crate_name(crates::name).like(pattern)); + } else if let Some(kw) = params.get("keyword") { + query = query.filter(crates::id.eq_any( + crates_keywords::table.select(crates_keywords::crate_id) + .inner_join(keywords::table) + .filter(lower(keywords::keyword).eq(lower(kw))) + )); + } else if let Some(cat) = params.get("category") { + query = query.filter(crates::id.eq_any( + crates_categories::table.select(crates_categories::crate_id) + .inner_join(categories::table) + .filter(split_part(categories::slug, "::", 1).eq(cat)) + )); + } else if let Some(user_id) = params.get("user_id").and_then(|s| s.parse::().ok()) { + query = query.filter(crates::id.eq_any(( + crate_owners::table.select(crate_owners::crate_id) + .filter(crate_owners::owner_id.eq(user_id)) + .filter(crate_owners::owner_kind.eq(OwnerKind::User as i32)) + ))); + } else if params.get("following").is_some() { + query = query.filter(crates::id.eq_any(( + follows::table.select(follows::crate_id) + .filter(follows::user_id.eq(user?.id)) + ))); } - args.insert(0, &id); - } else if needs_pattern { - args.insert(0, &pattern); + Ok(query) } - // Collect all the crates - let stmt = conn.prepare(&q)?; - let mut crates = Vec::new(); - for row in stmt.query(&args)?.iter() { - let krate: Crate = Model::from_row(&row); - let badges = krate.badges(conn)?; - let max_version = krate.max_version(conn)?; - crates.push(krate.minimal_encodable(max_version, Some(badges))); + let mut query = crates_query(¶ms, req.user())?; + let sort = params.get("sort").map(|s| &**s).unwrap_or("alpha"); + match (params.get("q"), sort) { + (Some(q), _) => { + let q = plainto_tsquery(q); + let rank = ts_rank_cd(crates::textsearchable_index_col, q); + query = query.order(rank.desc()) + } + (None, "downloads") => query = query.order(crates::downloads.desc()), + _ => query = query.order(crates::name.asc()), } - // Query for the total count of crates - let stmt = conn.prepare(&cnt)?; - let args = if args.len() > 2 {&args[..1]} else {&args[..0]}; - let rows = stmt.query(args)?; - let row = rows.iter().next().unwrap(); - let total = row.get(0); + let crates = query.limit(limit).offset(offset).load::(conn)?; + let versions = Version::belonging_to(&crates) + .load::(conn)? + .grouped_by(&crates) + .into_iter() + .map(|versions| { + versions.into_iter() + .map(|v| v.num) + .max() + .unwrap_or_else(|| semver::Version::parse("0.0.0").unwrap()) + }); + + let crates = versions.zip(crates).map(|(max_version, krate)| { + // FIXME: If we add crate_id to the Badge enum we can eliminate + // this N+1 + let badges = badges::table.filter(badges::crate_id.eq(krate.id)) + .load::(conn)?; + Ok(krate.minimal_encodable(max_version, Some(badges))) + }).collect::>()?; + + let total = crates_query(¶ms, req.user())?.count().get_result(conn)?; #[derive(RustcEncodable)] struct R { crates: Vec, meta: Meta } @@ -1201,3 +1165,8 @@ pub fn reverse_dependencies(req: &mut Request) -> CargoResult { struct Meta { total: i64 } Ok(req.json(&R{ dependencies: rev_deps, meta: Meta { total: total } })) } + +use diesel::types::{Text, Integer}; +sql_function!(canon_crate_name, canon_crate_name_t, (x: Text) -> Text); +sql_function!(lower, lower_t, (x: Text) -> Text); +sql_function!(split_part, split_part_t, (x: Text, y: Text, z: Integer) -> Text); diff --git a/src/lib.rs b/src/lib.rs index d555e19a8e2..440ab3cbb3f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,6 +23,7 @@ extern crate r2d2_postgres; extern crate rand; extern crate s3; extern crate semver; +extern crate serde_json; extern crate time; extern crate url; extern crate toml; diff --git a/src/schema.rs b/src/schema.rs index 8f1eab24017..8f88a8479f1 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -49,12 +49,11 @@ table! { updated_at -> Timestamp, created_at -> Timestamp, downloads -> Int4, - max_version -> Nullable, description -> Nullable, homepage -> Nullable, documentation -> Nullable, readme -> Nullable, - textsearchable_index_col -> Nullable<::diesel_full_text_search::TsVector>, + textsearchable_index_col -> ::diesel_full_text_search::TsVector, license -> Nullable, repository -> Nullable, max_upload_size -> Nullable, @@ -166,6 +165,6 @@ table! { created_at -> Timestamp, downloads -> Int4, features -> Nullable, - yanked -> Nullable, + yanked -> Bool, } } diff --git a/src/version.rs b/src/version.rs index a22eab42c6a..76ec30113f7 100644 --- a/src/version.rs +++ b/src/version.rs @@ -10,18 +10,22 @@ use time::Duration; use time::Timespec; use url; -use {Model, Crate}; use app::RequestApp; use db::RequestTransaction; +use diesel::prelude::*; +use diesel::pg::Pg; use dependency::{Dependency, EncodableDependency, Kind}; use download::{VersionDownload, EncodableVersionDownload}; use git; +use owner::{rights, Rights}; +use schema::versions; use upload; use user::RequestUser; -use owner::{rights, Rights}; use util::{RequestUtils, CargoResult, ChainError, internal, human}; +use {Model, Crate}; -#[derive(Clone)] +#[derive(Clone, Identifiable, Associations)] +#[belongs_to(Crate)] pub struct Version { pub id: i32, pub crate_id: i32, @@ -190,6 +194,26 @@ impl Version { } } +impl Queryable for Version { + type Row = (i32, i32, String, Timespec, Timespec, i32, Option, bool); + + fn build(row: Self::Row) -> Self { + let features = row.6.map(|s| { + json::decode(&s).unwrap() + }).unwrap_or_else(|| HashMap::new()); + Version { + id: row.0, + crate_id: row.1, + num: semver::Version::parse(&row.2).unwrap(), + updated_at: row.3, + created_at: row.4, + downloads: row.5, + features: features, + yanked: row.7, + } + } +} + impl Model for Version { fn from_row(row: &Row) -> Version { let num: String = row.get("num");