diff --git a/src/db/add_package.rs b/src/db/add_package.rs index 7ec1b7049..3658d4ed9 100644 --- a/src/db/add_package.rs +++ b/src/db/add_package.rs @@ -1,23 +1,22 @@ - -use crate::utils::MetadataPackage; use crate::docbuilder::BuildResult; +use crate::utils::MetadataPackage; use regex::Regex; +use std::fs; use std::io::prelude::*; use std::io::BufReader; use std::path::Path; -use std::fs; -use time::Timespec; -use rustc_serialize::json::{Json, ToJson}; -use slug::slugify; -use reqwest::Client; +use crate::error::Result; +use failure::err_msg; +use postgres::Connection; use reqwest::header::ACCEPT; +use reqwest::Client; +use rustc_serialize::json::{Json, ToJson}; use semver::Version; -use postgres::Connection; +use slug::slugify; use time; -use crate::error::Result; -use failure::err_msg; +use time::Timespec; /// Adds a package into database. /// @@ -25,17 +24,19 @@ use failure::err_msg; /// /// NOTE: `source_files` refers to the files originally in the crate, /// not the files generated by rustdoc. -pub(crate) fn add_package_into_database(conn: &Connection, - metadata_pkg: &MetadataPackage, - source_dir: &Path, - res: &BuildResult, - default_target: &str, - source_files: Option, - doc_targets: Vec, - cratesio_data: &CratesIoData, - has_docs: bool, - has_examples: bool) - -> Result { +// REFACTOR: This should be multiple smaller functions +pub(crate) fn add_package_into_database( + conn: &Connection, + metadata_pkg: &MetadataPackage, + source_dir: &Path, + res: &BuildResult, + default_target: &str, + source_files: Option, + doc_targets: Vec, + cratesio_data: &CratesIoData, + has_docs: bool, + has_examples: bool, +) -> Result { debug!("Adding package into database"); let crate_id = initialize_package_in_database(&conn, metadata_pkg)?; let dependencies = convert_dependencies(metadata_pkg); @@ -44,11 +45,15 @@ pub(crate) fn add_package_into_database(conn: &Connection, let is_library = metadata_pkg.is_library(); let release_id: i32 = { - let rows = conn.query("SELECT id FROM releases WHERE crate_id = $1 AND version = $2", - &[&crate_id, &format!("{}", metadata_pkg.version)])?; - - if rows.len() == 0 { - let rows = conn.query("INSERT INTO releases ( + let rows = conn.query( + "SELECT id FROM releases WHERE crate_id = $1 AND version = $2", + &[&crate_id, &metadata_pkg.version], + )?; + + if rows.is_empty() { + // REFACTOR: These are absurdly large + let rows = conn.query( + "INSERT INTO releases ( crate_id, version, release_time, dependencies, target_name, yanked, build_status, rustdoc_status, test_status, license, repository_url, @@ -62,36 +67,40 @@ pub(crate) fn add_package_into_database(conn: &Connection, $20, $21, $22, $23, $24, $25 ) RETURNING id", - &[&crate_id, - &metadata_pkg.version, - &cratesio_data.release_time, - &dependencies.to_json(), - &metadata_pkg.package_name(), - &cratesio_data.yanked, - &res.successful, - &has_docs, - &false, // TODO: Add test status somehow - &metadata_pkg.license, - &metadata_pkg.repository, - &metadata_pkg.homepage, - &metadata_pkg.description, - &rustdoc, - &readme, - &metadata_pkg.authors.to_json(), - &metadata_pkg.keywords.to_json(), - &has_examples, - &cratesio_data.downloads, - &source_files, - &doc_targets.to_json(), - &is_library, - &res.rustc_version, - &metadata_pkg.documentation, - &default_target])?; + &[ + &crate_id, + &metadata_pkg.version, + &cratesio_data.release_time, + &dependencies.to_json(), + &metadata_pkg.package_name(), + &cratesio_data.yanked, + &res.successful, + &has_docs, + &false, // TODO: Add test status somehow + &metadata_pkg.license, + &metadata_pkg.repository, + &metadata_pkg.homepage, + &metadata_pkg.description, + &rustdoc, + &readme, + &metadata_pkg.authors.to_json(), + &metadata_pkg.keywords.to_json(), + &has_examples, + &cratesio_data.downloads, + &source_files, + &doc_targets.to_json(), + &is_library, + &res.rustc_version, + &metadata_pkg.documentation, + &default_target, + ], + )?; // return id rows.get(0).get(0) - } else { - conn.query("UPDATE releases + // REFACTOR: These are absurdly large + conn.query( + "UPDATE releases SET release_time = $3, dependencies = $4, target_name = $5, @@ -116,100 +125,115 @@ pub(crate) fn add_package_into_database(conn: &Connection, documentation_url = $24, default_target = $25 WHERE crate_id = $1 AND version = $2", - &[&crate_id, - &format!("{}", metadata_pkg.version), - &cratesio_data.release_time, - &dependencies.to_json(), - &metadata_pkg.package_name(), - &cratesio_data.yanked, - &res.successful, - &has_docs, - &false, // TODO: Add test status somehow - &metadata_pkg.license, - &metadata_pkg.repository, - &metadata_pkg.homepage, - &metadata_pkg.description, - &rustdoc, - &readme, - &metadata_pkg.authors.to_json(), - &metadata_pkg.keywords.to_json(), - &has_examples, - &cratesio_data.downloads, - &source_files, - &doc_targets.to_json(), - &is_library, - &res.rustc_version, - &metadata_pkg.documentation, - &default_target])?; + &[ + &crate_id, + &metadata_pkg.version, + &cratesio_data.release_time, + &dependencies.to_json(), + &metadata_pkg.package_name(), + &cratesio_data.yanked, + &res.successful, + &has_docs, + &false, // TODO: Add test status somehow + &metadata_pkg.license, + &metadata_pkg.repository, + &metadata_pkg.homepage, + &metadata_pkg.description, + &rustdoc, + &readme, + &metadata_pkg.authors.to_json(), + &metadata_pkg.keywords.to_json(), + &has_examples, + &cratesio_data.downloads, + &source_files, + &doc_targets.to_json(), + &is_library, + &res.rustc_version, + &metadata_pkg.documentation, + &default_target, + ], + )?; rows.get(0).get(0) } }; - - add_keywords_into_database(&conn, &metadata_pkg, &release_id)?; - add_authors_into_database(&conn, &metadata_pkg, &release_id)?; - add_owners_into_database(&conn, &cratesio_data.owners, &crate_id)?; - + add_keywords_into_database(&conn, &metadata_pkg, release_id)?; + add_authors_into_database(&conn, &metadata_pkg, release_id)?; + add_owners_into_database(&conn, &cratesio_data.owners, crate_id)?; // Update versions { let metadata_version = Version::parse(&metadata_pkg.version)?; - let mut versions: Json = conn.query("SELECT versions FROM crates WHERE id = $1", - &[&crate_id])? + let mut versions: Json = conn + .query("SELECT versions FROM crates WHERE id = $1", &[&crate_id])? .get(0) .get(0); + if let Some(versions_array) = versions.as_array_mut() { let mut found = false; + for version in versions_array.clone() { let version = Version::parse(version.as_string().unwrap())?; + if version == metadata_version { found = true; } } + if !found { - versions_array.push(format!("{}", &metadata_pkg.version).to_json()); + versions_array.push(metadata_pkg.version.to_json()); } } - let _ = conn.query("UPDATE crates SET versions = $1 WHERE id = $2", - &[&versions, &crate_id]); + + let _ = conn.query( + "UPDATE crates SET versions = $1 WHERE id = $2", + &[&versions, &crate_id], + ); } Ok(release_id) } - /// Adds a build into database -pub(crate) fn add_build_into_database(conn: &Connection, - release_id: &i32, - res: &BuildResult) - -> Result { +pub(crate) fn add_build_into_database( + conn: &Connection, + release_id: i32, + res: &BuildResult, +) -> Result { debug!("Adding build into database"); - let rows = conn.query("INSERT INTO builds (rid, rustc_version, - cratesfyi_version, - build_status, output) - VALUES ($1, $2, $3, $4, $5) - RETURNING id", - &[release_id, - &res.rustc_version, - &res.docsrs_version, - &res.successful, - &res.build_log])?; + + let rows = conn.query( + "INSERT INTO builds (rid, rustc_version, + cratesfyi_version, + build_status, output) + VALUES ($1, $2, $3, $4, $5) + RETURNING id", + &[ + &release_id, + &res.rustc_version, + &res.docsrs_version, + &res.successful, + &res.build_log, + ], + )?; + Ok(rows.get(0).get(0)) } - fn initialize_package_in_database(conn: &Connection, pkg: &MetadataPackage) -> Result { let mut rows = conn.query("SELECT id FROM crates WHERE name = $1", &[&pkg.name])?; + // insert crate into database if it is not exists - if rows.len() == 0 { - rows = conn.query("INSERT INTO crates (name) VALUES ($1) RETURNING id", - &[&pkg.name])?; + if rows.is_empty() { + rows = conn.query( + "INSERT INTO crates (name) VALUES ($1) RETURNING id", + &[&pkg.name], + )?; } + Ok(rows.get(0).get(0)) } - - /// Convert dependencies into Vec<(String, String, String)> fn convert_dependencies(pkg: &MetadataPackage) -> Vec<(String, String, String)> { let mut dependencies: Vec<(String, String, String)> = Vec::new(); @@ -219,32 +243,34 @@ fn convert_dependencies(pkg: &MetadataPackage) -> Vec<(String, String, String)> let kind = dependency.kind.clone().unwrap_or_else(|| "normal".into()); dependencies.push((name, version, kind.to_string())); } + dependencies } - /// Reads readme if there is any read defined in Cargo.toml of a Package fn get_readme(pkg: &MetadataPackage, source_dir: &Path) -> Result> { - let readme_path = source_dir.join(pkg.readme.clone().unwrap_or("README.md".to_owned())); + let readme_path = source_dir.join(pkg.readme.clone().unwrap_or_else(|| "README.md".to_owned())); if !readme_path.exists() { return Ok(None); } - let mut reader = fs::File::open(readme_path).map(|f| BufReader::new(f))?; + let mut reader = fs::File::open(readme_path).map(BufReader::new)?; let mut readme = String::new(); reader.read_to_string(&mut readme)?; if readme.is_empty() { Ok(None) } else if readme.len() > 51200 { - Ok(Some(format!("(Readme ignored due to being too long. ({} > 51200))", readme.len()))) + Ok(Some(format!( + "(Readme ignored due to being too long. ({} > 51200))", + readme.len() + ))) } else { Ok(Some(readme)) } } - fn get_rustdoc(pkg: &MetadataPackage, source_dir: &Path) -> Result> { if let Some(src_path) = &pkg.targets[0].src_path { let src_path = Path::new(src_path); @@ -259,20 +285,22 @@ fn get_rustdoc(pkg: &MetadataPackage, source_dir: &Path) -> Result Result> { - let reader = fs::File::open(file_path).map(|f| BufReader::new(f))?; + let reader = fs::File::open(file_path).map(BufReader::new)?; let mut rustdoc = String::new(); for line in reader.lines() { let line = line?; + if line.starts_with("//!") { // some lines may or may not have a space between the `//!` and the start of the text let line = line.trim_start_matches("//!").trim_start(); + if !line.is_empty() { rustdoc.push_str(line); } + rustdoc.push('\n'); } } @@ -280,13 +308,15 @@ fn read_rust_doc(file_path: &Path) -> Result> { if rustdoc.is_empty() { Ok(None) } else if rustdoc.len() > 51200 { - Ok(Some(format!("(Library doc comment ignored due to being too long. ({} > 51200))", rustdoc.len()))) + Ok(Some(format!( + "(Library doc comment ignored due to being too long. ({} > 51200))", + rustdoc.len() + ))) } else { Ok(Some(rustdoc)) } } - pub(crate) struct CratesIoData { pub(crate) release_time: Timespec, pub(crate) yanked: bool, @@ -308,22 +338,25 @@ impl CratesIoData { } } - /// Get release_time, yanked and downloads from crates.io -fn get_release_time_yanked_downloads( - pkg: &MetadataPackage, -) -> Result<(time::Timespec, bool, i32)> { +fn get_release_time_yanked_downloads(pkg: &MetadataPackage) -> Result<(time::Timespec, bool, i32)> { let url = format!("https://crates.io/api/v1/crates/{}/versions", pkg.name); + // FIXME: There is probably better way to do this // and so many unwraps... let client = Client::new(); - let mut res = client.get(&url[..]) + + let mut res = client + .get(&url[..]) .header(ACCEPT, "application/json") .send()?; + let mut body = String::new(); res.read_to_string(&mut body).unwrap(); let json = Json::from_str(&body[..]).unwrap(); - let versions = json.as_object() + + let versions = json + .as_object() .and_then(|o| o.get("versions")) .and_then(|v| v.as_array()) .ok_or_else(|| err_msg("Not a JSON object"))?; @@ -331,93 +364,132 @@ fn get_release_time_yanked_downloads( let (mut release_time, mut yanked, mut downloads) = (None, None, None); for version in versions { - let version = version.as_object().ok_or_else(|| err_msg("Not a JSON object"))?; - let version_num = version.get("num") + let version = version + .as_object() + .ok_or_else(|| err_msg("Not a JSON object"))?; + + let version_num = version + .get("num") .and_then(|v| v.as_string()) .ok_or_else(|| err_msg("Not a JSON object"))?; if semver::Version::parse(version_num).unwrap().to_string() == pkg.version { - let release_time_raw = version.get("created_at") + let release_time_raw = version + .get("created_at") .and_then(|c| c.as_string()) .ok_or_else(|| err_msg("Not a JSON object"))?; - release_time = Some(time::strptime(release_time_raw, "%Y-%m-%dT%H:%M:%S") - .unwrap() - .to_timespec()); - yanked = Some(version.get("yanked") - .and_then(|c| c.as_boolean()) - .ok_or_else(|| err_msg("Not a JSON object"))?); - - downloads = Some(version.get("downloads") - .and_then(|c| c.as_i64()) - .ok_or_else(|| err_msg("Not a JSON object"))? as i32); + release_time = Some( + time::strptime(release_time_raw, "%Y-%m-%dT%H:%M:%S") + .unwrap() + .to_timespec(), + ); + + yanked = Some( + version + .get("yanked") + .and_then(|c| c.as_boolean()) + .ok_or_else(|| err_msg("Not a JSON object"))?, + ); + + downloads = Some( + version + .get("downloads") + .and_then(|c| c.as_i64()) + .ok_or_else(|| err_msg("Not a JSON object"))? as i32, + ); break; } } - Ok((release_time.unwrap_or(time::get_time()), yanked.unwrap_or(false), downloads.unwrap_or(0))) + Ok(( + release_time.unwrap_or_else(time::get_time), + yanked.unwrap_or(false), + downloads.unwrap_or(0), + )) } - /// Adds keywords into database -fn add_keywords_into_database(conn: &Connection, pkg: &MetadataPackage, release_id: &i32) -> Result<()> { +fn add_keywords_into_database( + conn: &Connection, + pkg: &MetadataPackage, + release_id: i32, +) -> Result<()> { for keyword in &pkg.keywords { let slug = slugify(&keyword); let keyword_id: i32 = { let rows = conn.query("SELECT id FROM keywords WHERE slug = $1", &[&slug])?; - if rows.len() > 0 { + + if !rows.is_empty() { rows.get(0).get(0) } else { - conn.query("INSERT INTO keywords (name, slug) VALUES ($1, $2) RETURNING id", - &[&keyword, &slug])? - .get(0) - .get(0) + conn.query( + "INSERT INTO keywords (name, slug) VALUES ($1, $2) RETURNING id", + &[&keyword, &slug], + )? + .get(0) + .get(0) } }; + // add releationship - let _ = conn.query("INSERT INTO keyword_rels (rid, kid) VALUES ($1, $2)", - &[release_id, &keyword_id]); + let _ = conn.query( + "INSERT INTO keyword_rels (rid, kid) VALUES ($1, $2)", + &[&release_id, &keyword_id], + ); } Ok(()) } - - /// Adds authors into database -fn add_authors_into_database(conn: &Connection, pkg: &MetadataPackage, release_id: &i32) -> Result<()> { - +fn add_authors_into_database( + conn: &Connection, + pkg: &MetadataPackage, + release_id: i32, +) -> Result<()> { let author_capture_re = Regex::new("^([^><]+)<*(.*?)>*$").unwrap(); for author in &pkg.authors { if let Some(author_captures) = author_capture_re.captures(&author[..]) { - let author = author_captures.get(1).map(|m| m.as_str()).unwrap_or("").trim(); - let email = author_captures.get(2).map(|m| m.as_str()).unwrap_or("").trim(); + let author = author_captures + .get(1) + .map(|m| m.as_str()) + .unwrap_or("") + .trim(); + let email = author_captures + .get(2) + .map(|m| m.as_str()) + .unwrap_or("") + .trim(); let slug = slugify(&author); let author_id: i32 = { let rows = conn.query("SELECT id FROM authors WHERE slug = $1", &[&slug])?; - if rows.len() > 0 { + if !rows.is_empty() { rows.get(0).get(0) } else { - conn.query("INSERT INTO authors (name, email, slug) VALUES ($1, $2, $3) + conn.query( + "INSERT INTO authors (name, email, slug) VALUES ($1, $2, $3) RETURNING id", - &[&author, &email, &slug])? - .get(0) - .get(0) + &[&author, &email, &slug], + )? + .get(0) + .get(0) } }; // add relationship - let _ = conn.query("INSERT INTO author_rels (rid, aid) VALUES ($1, $2)", - &[release_id, &author_id]); + let _ = conn.query( + "INSERT INTO author_rels (rid, aid) VALUES ($1, $2)", + &[&release_id, &author_id], + ); } } Ok(()) } - pub(crate) struct CrateOwner { pub(crate) avatar: String, pub(crate) email: String, @@ -430,7 +502,8 @@ fn get_owners(pkg: &MetadataPackage) -> Result> { // owners available in: https://crates.io/api/v1/crates/rand/owners let owners_url = format!("https://crates.io/api/v1/crates/{}/owners", pkg.name); let client = Client::new(); - let mut res = client.get(&owners_url[..]) + let mut res = client + .get(&owners_url[..]) .header(ACCEPT, "application/json") .send()?; // FIXME: There is probably better way to do this @@ -440,24 +513,33 @@ fn get_owners(pkg: &MetadataPackage) -> Result> { let json = Json::from_str(&body[..])?; let mut result = Vec::new(); - if let Some(owners) = json.as_object() + if let Some(owners) = json + .as_object() .and_then(|j| j.get("users")) - .and_then(|j| j.as_array()) { + .and_then(|j| j.as_array()) + { for owner in owners { // FIXME: I know there is a better way to do this - let avatar = owner.as_object() + let avatar = owner + .as_object() .and_then(|o| o.get("avatar")) .and_then(|o| o.as_string()) .unwrap_or(""); - let email = owner.as_object() + + let email = owner + .as_object() .and_then(|o| o.get("email")) .and_then(|o| o.as_string()) .unwrap_or(""); - let login = owner.as_object() + + let login = owner + .as_object() .and_then(|o| o.get("login")) .and_then(|o| o.as_string()) .unwrap_or(""); - let name = owner.as_object() + + let name = owner + .as_object() .and_then(|o| o.get("name")) .and_then(|o| o.as_string()) .unwrap_or(""); @@ -479,25 +561,29 @@ fn get_owners(pkg: &MetadataPackage) -> Result> { } /// Adds owners into database -fn add_owners_into_database(conn: &Connection, owners: &[CrateOwner], crate_id: &i32) -> Result<()> { +fn add_owners_into_database(conn: &Connection, owners: &[CrateOwner], crate_id: i32) -> Result<()> { for owner in owners { let owner_id: i32 = { let rows = conn.query("SELECT id FROM owners WHERE login = $1", &[&owner.login])?; - if rows.len() > 0 { + if rows.is_empty() { rows.get(0).get(0) } else { - conn.query("INSERT INTO owners (login, avatar, name, email) + conn.query( + "INSERT INTO owners (login, avatar, name, email) VALUES ($1, $2, $3, $4) RETURNING id", - &[&owner.login, &owner.avatar, &owner.name, &owner.email])? - .get(0) - .get(0) + &[&owner.login, &owner.avatar, &owner.name, &owner.email], + )? + .get(0) + .get(0) } }; // add relationship - let _ = conn.query("INSERT INTO owner_rels (cid, oid) VALUES ($1, $2)", - &[crate_id, &owner_id]); + let _ = conn.query( + "INSERT INTO owner_rels (cid, oid) VALUES ($1, $2)", + &[&crate_id, &owner_id], + ); } Ok(()) } diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index d1175dfb7..880ac2064 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -389,7 +389,7 @@ impl RustwideBuilder { has_docs, has_examples, )?; - add_build_into_database(&conn, &release_id, &res.result)?; + add_build_into_database(&conn, release_id, &res.result)?; doc_builder.add_to_cache(name, version); Ok(res) diff --git a/src/test/fakes.rs b/src/test/fakes.rs index 998f96958..0894b5b24 100644 --- a/src/test/fakes.rs +++ b/src/test/fakes.rs @@ -182,7 +182,7 @@ impl<'a> FakeRelease<'a> { self.has_docs, self.has_examples, )?; - crate::db::add_build_into_database(&db.conn(), &release_id, &self.build_result)?; + crate::db::add_build_into_database(&db.conn(), release_id, &self.build_result)?; Ok(release_id) }