From 90b02dcd2f2c825cee1c9338e12e2fb9cda78949 Mon Sep 17 00:00:00 2001 From: Allstreamer <48365544+Allstreamer@users.noreply.github.com> Date: Mon, 8 Aug 2022 22:09:48 +0200 Subject: [PATCH 1/8] Implemented Warehouse Pypi API Call - Added log 0.4 Crate - Added serde 1.0 Crate - Added serde_json 1.0 Crate - Created pypi.rs - Created request_package_info function - Created PypiData struct - Added request_package_info example to main.rs --- Cargo.toml | 3 +++ src/main.rs | 15 ++++++++++++++- src/pypi.rs | 29 +++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 src/pypi.rs diff --git a/Cargo.toml b/Cargo.toml index 6524c79..2e022df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,9 @@ reqwest = { version = "0.11", features = ["blocking", "json"] } structopt = { version = "0.3.26", features = ["color"] } strum = "0.24.1" strum_macros = "0.24.2" +log = "0.4" +serde = {version = "1", features = ["derive"]} +serde_json = "1.0" [dev-dependencies] diff --git a/src/main.rs b/src/main.rs index fa54e9f..612684b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,9 @@ use std::path::PathBuf; use structopt::StructOpt; +use anyhow::Result; + +mod pypi; +use pypi::{request_package_info, PypiData}; /// A basic example #[derive(StructOpt, Debug)] @@ -44,7 +48,16 @@ enum Opt { Help {}, } -fn download_package(package_name: String, package_index: &String) {} +fn download_package(package_name: String, package_index: &String) -> Result<()> { + let package_info: PypiData = request_package_info(&package_name, &package_index)?; + + // Example of getting data this will be more robust as the + // PypiData struct gets expanded (meaning less calls to .get()) + let latest_version = package_info.info.get("version").unwrap(); + println!("Latest Version of {} is {}", package_name, latest_version); + + Ok(()) +} fn main() { let opt = Opt::from_args(); diff --git a/src/pypi.rs b/src/pypi.rs new file mode 100644 index 0000000..e47956a --- /dev/null +++ b/src/pypi.rs @@ -0,0 +1,29 @@ +use log::info; +use serde::{Deserialize, Serialize}; +use std::fmt::Display; + +/// +/// TODO: Implement more specific structs +#[derive(Debug, Serialize, Deserialize)] +pub struct PypiData { + pub info: serde_json::value::Value, + pub last_serial: i32, + pub releases: serde_json::value::Value, + pub urls: Vec, + pub vulnerabilities: Vec, +} + +/// Implements Warehouse Pypi API call & JSON conversion +pub fn request_package_info(package_name: T, package_index: T) -> Result +where + T: ToString + Display, +{ + let path = format!("{}/pypi/{}/json", package_index, package_name); + + info!("Requesting data from {}", path); + let resp: reqwest::blocking::Response = reqwest::blocking::get(path)?; + + let decoded_json: PypiData = resp.json()?; + + Ok(decoded_json) +} From 605ae334ae41cf8439eae8b3f48320f0b2b07665 Mon Sep 17 00:00:00 2001 From: Allstreamer <48365544+Allstreamer@users.noreply.github.com> Date: Mon, 8 Aug 2022 22:36:20 +0200 Subject: [PATCH 2/8] Formatting & Linting - Solved clippy suggestions - Ran cargo fmt --- Cargo.lock | 119 +++++++++++++++++++++++++++++++++++++++++++++++++--- src/main.rs | 10 ++--- src/pypi.rs | 7 +++- 3 files changed, 123 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c897e7..c9940e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anyhow" version = "1.0.59" @@ -94,6 +103,21 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width", + "vec_map", +] + [[package]] name = "clap" version = "3.2.16" @@ -106,9 +130,9 @@ dependencies = [ "clap_lex", "indexmap", "once_cell", - "strsim", + "strsim 0.10.0", "termcolor", - "textwrap", + "textwrap 0.15.0", ] [[package]] @@ -117,7 +141,7 @@ version = "3.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba52acd3b0a5c33aeada5cdaa3267cdc7c594a98731d4268cdc1532f4264cb4" dependencies = [ - "heck", + "heck 0.4.0", "proc-macro-error", "proc-macro2", "quote", @@ -277,6 +301,15 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "heck" version = "0.4.0" @@ -694,9 +727,14 @@ name = "rust-pip" version = "0.0.1" dependencies = [ "anyhow", - "clap", + "clap 3.2.16", + "log", "reqwest", + "serde", + "serde_json", + "structopt", "strum", + "strum_macros", ] [[package]] @@ -755,6 +793,20 @@ name = "serde" version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc855a42c7967b7c369eb5860f7164ef1f6f81c20c7cc1141f2a604e18723b03" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f2122636b9fe3b81f1cb25099fcf2d3f542cdb1d45940d56c713158884a05da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "serde_json" @@ -798,12 +850,42 @@ dependencies = [ "winapi", ] +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap 2.34.0", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck 0.3.3", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "strum" version = "0.24.1" @@ -819,7 +901,7 @@ version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4faebde00e8ff94316c01800f9054fd2ba77d30d9e922541913051d1d978918b" dependencies = [ - "heck", + "heck 0.4.0", "proc-macro2", "quote", "rustversion", @@ -860,6 +942,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "textwrap" version = "0.15.0" @@ -976,6 +1067,18 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + [[package]] name = "url" version = "2.2.2" @@ -994,6 +1097,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" version = "0.9.4" diff --git a/src/main.rs b/src/main.rs index dcf0fa7..dd3f769 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,5 @@ -use std::path::PathBuf; -use structopt::StructOpt; -use clap::{AppSettings, Parser}; use anyhow::Result; +use clap::{AppSettings, Parser}; mod pypi; use pypi::{request_package_info, PypiData}; @@ -50,9 +48,9 @@ enum Opt { } fn download_package(package_name: String, package_index: &String) -> Result<()> { - let package_info: PypiData = request_package_info(&package_name, &package_index)?; + let package_info: PypiData = request_package_info(&package_name, package_index)?; - // Example of getting data this will be more robust as the + // Example of getting data this will be more robust as the // PypiData struct gets expanded (meaning less calls to .get()) let latest_version = package_info.info.get("version").unwrap(); println!("Latest Version of {} is {}", package_name, latest_version); @@ -68,7 +66,7 @@ fn main() { Opt::Download { name, index } => { println!("Package name {:?}", name); println!("Index name: {:?}", index); - download_package(name, &index); + let _ = download_package(name, &index); } _ => todo!(), } diff --git a/src/pypi.rs b/src/pypi.rs index e47956a..0d77913 100644 --- a/src/pypi.rs +++ b/src/pypi.rs @@ -2,7 +2,7 @@ use log::info; use serde::{Deserialize, Serialize}; use std::fmt::Display; -/// +/// /// TODO: Implement more specific structs #[derive(Debug, Serialize, Deserialize)] pub struct PypiData { @@ -14,7 +14,10 @@ pub struct PypiData { } /// Implements Warehouse Pypi API call & JSON conversion -pub fn request_package_info(package_name: T, package_index: T) -> Result +pub fn request_package_info( + package_name: T, + package_index: T, +) -> Result where T: ToString + Display, { From 85de44b05e8d79701dce33ca626dfdf34f05bf9e Mon Sep 17 00:00:00 2001 From: Allstreamer <48365544+Allstreamer@users.noreply.github.com> Date: Tue, 9 Aug 2022 10:59:21 +0200 Subject: [PATCH 3/8] Updated pypi module - Added Docstrings - Renamed PypiData to PyPIData - Added unit tests --- src/main.rs | 6 +++--- src/pypi.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/main.rs b/src/main.rs index dd3f769..80b7963 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,7 @@ use anyhow::Result; use clap::{AppSettings, Parser}; mod pypi; -use pypi::{request_package_info, PypiData}; +use pypi::{request_package_info, PyPIData}; /// Python package manager written in Rust #[derive(Parser, Debug)] @@ -48,10 +48,10 @@ enum Opt { } fn download_package(package_name: String, package_index: &String) -> Result<()> { - let package_info: PypiData = request_package_info(&package_name, package_index)?; + let package_info: PyPIData = request_package_info(&package_name, package_index)?; // Example of getting data this will be more robust as the - // PypiData struct gets expanded (meaning less calls to .get()) + // PyPIData struct gets expanded (meaning less calls to .get()) let latest_version = package_info.info.get("version").unwrap(); println!("Latest Version of {} is {}", package_name, latest_version); diff --git a/src/pypi.rs b/src/pypi.rs index 0d77913..d8dded7 100644 --- a/src/pypi.rs +++ b/src/pypi.rs @@ -1,23 +1,37 @@ +//! Warehouse PyPI API Implementation + use log::info; use serde::{Deserialize, Serialize}; use std::fmt::Display; -/// -/// TODO: Implement more specific structs +/// Set of Information describing a Python package hosted on a Warehouse instance +/// for exact details of what is contained go to https://warehouse.pypa.io/api-reference/json.html#project #[derive(Debug, Serialize, Deserialize)] -pub struct PypiData { +pub struct PyPIData { + /// Contains data such as Package Name, Author, pub info: serde_json::value::Value, pub last_serial: i32, + /// List of releases containing data such as pub releases: serde_json::value::Value, + /// Link and related data to sdist & bdist_wheel pub urls: Vec, + /// Vector of known vulnerabilities of the package pub vulnerabilities: Vec, } -/// Implements Warehouse Pypi API call & JSON conversion +/// Implements Warehouse PyPI API call & JSON conversion +/// +/// # Example +/// ``` +/// use rust-pip::PyPI::request_package_info; +/// +/// let data = request_package_info("numpy", "https://pypi.org/").unwrap(); +/// assert_eq!(data.info.get("license").unwrap(), "BSD"); +/// ``` pub fn request_package_info( package_name: T, package_index: T, -) -> Result +) -> Result where T: ToString + Display, { @@ -26,7 +40,38 @@ where info!("Requesting data from {}", path); let resp: reqwest::blocking::Response = reqwest::blocking::get(path)?; - let decoded_json: PypiData = resp.json()?; + let decoded_json: PyPIData = resp.json()?; Ok(decoded_json) } + +#[cfg(test)] +mod tests { + use crate::pypi::request_package_info; + + #[test] + fn check_numpy_licence() { + let data = request_package_info("numpy", "https://pypi.org/").unwrap(); + + assert_eq!(data.info.get("license").unwrap(), "BSD"); + } + + #[test] + fn check_pytorch_name() { + let data = request_package_info("pytorch", "https://pypi.org/").unwrap(); + + assert_eq!(data.info.get("name").unwrap(), "pytorch"); + } + + #[test] + fn check_pytorch_download_v1() { + let data = request_package_info("numpy", "https://pypi.org/").unwrap(); + + assert_eq!( + data.releases.get("1.0").unwrap()[0] + .get("filename") + .unwrap(), + "numpy-1.0.1.dev3460.win32-py2.4.exe" + ); + } +} From 052c9878662f6f81023c24234b91f7ceb0b14865 Mon Sep 17 00:00:00 2001 From: Allstreamer <48365544+Allstreamer@users.noreply.github.com> Date: Tue, 9 Aug 2022 11:35:35 +0200 Subject: [PATCH 4/8] Rename check_pytorch_download_v1 - Renamed check_pytorch_download_v1 to check_numpy_download_name_v1 --- src/pypi.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pypi.rs b/src/pypi.rs index d8dded7..af7f790 100644 --- a/src/pypi.rs +++ b/src/pypi.rs @@ -64,7 +64,7 @@ mod tests { } #[test] - fn check_pytorch_download_v1() { + fn check_numpy_download_name_v1() { let data = request_package_info("numpy", "https://pypi.org/").unwrap(); assert_eq!( From 50eecae693538866ceff575f18538502086d8487 Mon Sep 17 00:00:00 2001 From: Allstreamer <48365544+Allstreamer@users.noreply.github.com> Date: Tue, 9 Aug 2022 13:34:00 +0200 Subject: [PATCH 5/8] Updated pypi.rs to include negative test case - Added Negative test case (Haven't added others for .get() calls since those will be replaced soon with more specific structs) --- src/pypi.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/pypi.rs b/src/pypi.rs index af7f790..f44319d 100644 --- a/src/pypi.rs +++ b/src/pypi.rs @@ -6,7 +6,7 @@ use std::fmt::Display; /// Set of Information describing a Python package hosted on a Warehouse instance /// for exact details of what is contained go to https://warehouse.pypa.io/api-reference/json.html#project -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, PartialEq)] pub struct PyPIData { /// Contains data such as Package Name, Author, pub info: serde_json::value::Value, @@ -74,4 +74,11 @@ mod tests { "numpy-1.0.1.dev3460.win32-py2.4.exe" ); } + + #[test] + #[should_panic(expected = "`Err` value: reqwest::Error")] + fn check_fails_invalid_url() { + let _err = + request_package_info("numpy", "invalid_url obviously wrong").unwrap(); + } } From 3803f21e84eca0cec0ce00cbc849836577a94689 Mon Sep 17 00:00:00 2001 From: Allstreamer <48365544+Allstreamer@users.noreply.github.com> Date: Tue, 9 Aug 2022 14:08:51 +0200 Subject: [PATCH 6/8] Update pypi.rs - Fixed some doc strings --- src/pypi.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pypi.rs b/src/pypi.rs index f44319d..696e050 100644 --- a/src/pypi.rs +++ b/src/pypi.rs @@ -5,13 +5,13 @@ use serde::{Deserialize, Serialize}; use std::fmt::Display; /// Set of Information describing a Python package hosted on a Warehouse instance -/// for exact details of what is contained go to https://warehouse.pypa.io/api-reference/json.html#project +/// for exact details of what is contained go to #[derive(Debug, Serialize, Deserialize, PartialEq)] pub struct PyPIData { - /// Contains data such as Package Name, Author, + /// Contains data such as Package Name, Author and Licence pub info: serde_json::value::Value, pub last_serial: i32, - /// List of releases containing data such as + /// List of releases containing Object with downloads for each release and it's versions pub releases: serde_json::value::Value, /// Link and related data to sdist & bdist_wheel pub urls: Vec, From 2180227178a358576121414ed1d4cca91e6e11cf Mon Sep 17 00:00:00 2001 From: Allstreamer <48365544+Allstreamer@users.noreply.github.com> Date: Tue, 9 Aug 2022 14:28:32 +0200 Subject: [PATCH 7/8] Added PyPIPackageInfo & PyPIPackageDownloadInfo - Created PyPIPackageInfo & PyPIPackageDownloadInfo Structs - Updated unit-tests, examples & doc examples --- src/main.rs | 2 +- src/pypi.rs | 52 +++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/src/main.rs b/src/main.rs index 80b7963..634ddeb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -52,7 +52,7 @@ fn download_package(package_name: String, package_index: &String) -> Result<()> // Example of getting data this will be more robust as the // PyPIData struct gets expanded (meaning less calls to .get()) - let latest_version = package_info.info.get("version").unwrap(); + let latest_version = package_info.info.version; println!("Latest Version of {} is {}", package_name, latest_version); Ok(()) diff --git a/src/pypi.rs b/src/pypi.rs index 696e050..fab7465 100644 --- a/src/pypi.rs +++ b/src/pypi.rs @@ -4,12 +4,54 @@ use log::info; use serde::{Deserialize, Serialize}; use std::fmt::Display; +/// Download stats +#[derive(Debug, Serialize, Deserialize, PartialEq)] +pub struct PyPIPackageDownloadInfo { + last_day: i32, + last_week: i32, + last_month: i32, +} +/// Public package information +#[derive(Debug, Serialize, Deserialize, PartialEq)] +pub struct PyPIPackageInfo { + pub author: String, + pub author_email: String, + pub bugtrack_url: serde_json::value::Value, + pub classifiers: Vec, + pub description: String, + pub description_content_type: String, + pub docs_url: serde_json::value::Value, + pub download_url: String, + pub downloads: PyPIPackageDownloadInfo, + pub home_page: String, + pub keywords: String, + pub license: String, + pub maintainer: String, + pub maintainer_email: String, + /// Package name + pub name: String, + pub package_url: String, + pub platform: String, + pub project_url: String, + pub project_urls: serde_json::value::Value, + pub release_url: String, + pub requires_dist: serde_json::value::Value, + /// Minimum required python version + pub requires_python: String, + /// Project Summary + pub summary: String, + /// Latest stable version number + pub version: String, + pub yanked: bool, + pub yanked_reason: serde_json::value::Value, +} + /// Set of Information describing a Python package hosted on a Warehouse instance /// for exact details of what is contained go to #[derive(Debug, Serialize, Deserialize, PartialEq)] pub struct PyPIData { - /// Contains data such as Package Name, Author and Licence - pub info: serde_json::value::Value, + /// Contains data such as package name, author and license + pub info: PyPIPackageInfo, pub last_serial: i32, /// List of releases containing Object with downloads for each release and it's versions pub releases: serde_json::value::Value, @@ -26,7 +68,7 @@ pub struct PyPIData { /// use rust-pip::PyPI::request_package_info; /// /// let data = request_package_info("numpy", "https://pypi.org/").unwrap(); -/// assert_eq!(data.info.get("license").unwrap(), "BSD"); +/// assert_eq!(data.info.license, "BSD"); /// ``` pub fn request_package_info( package_name: T, @@ -53,14 +95,14 @@ mod tests { fn check_numpy_licence() { let data = request_package_info("numpy", "https://pypi.org/").unwrap(); - assert_eq!(data.info.get("license").unwrap(), "BSD"); + assert_eq!(data.info.license, "BSD"); } #[test] fn check_pytorch_name() { let data = request_package_info("pytorch", "https://pypi.org/").unwrap(); - assert_eq!(data.info.get("name").unwrap(), "pytorch"); + assert_eq!(data.info.name, "pytorch"); } #[test] From f42e6485775f8574ca7ee51348cf8f537a10f176 Mon Sep 17 00:00:00 2001 From: Allstreamer <48365544+Allstreamer@users.noreply.github.com> Date: Sun, 4 Sep 2022 18:32:35 +0200 Subject: [PATCH 8/8] Fixed Documentation --- src/pypi.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pypi.rs b/src/pypi.rs index fab7465..e3031f8 100644 --- a/src/pypi.rs +++ b/src/pypi.rs @@ -65,7 +65,7 @@ pub struct PyPIData { /// /// # Example /// ``` -/// use rust-pip::PyPI::request_package_info; +/// use pypi::request_package_info; /// /// let data = request_package_info("numpy", "https://pypi.org/").unwrap(); /// assert_eq!(data.info.license, "BSD");