Auto merge of #4548 - teozkr:feature/minimal-crate-metadata-api, r=jtgeibel

Add a "minimal crate metadata" API endpoint

As mentioned in #4503 and renovatebot/renovate#3486, this is intended to be a lighter-weight API than `/crates/:crate_id`, which can be subject to looser rate limits.

This adds a new `/crates/:crate_id/crate` endpoint for metadata about the crate itself, without information about related objects (such as versions or categories).

Author: bors

src/controllers/krate/metadata.rs

@@ -5,6 +5,7 @@
 //! `Cargo.toml` file.
 
 use std::cmp::Reverse;
+use std::str::FromStr;
 
 use crate::controllers::frontend_prelude::*;
 use crate::controllers::helpers::pagination::PaginationOptions;
@@ -45,7 +46,7 @@ pub fn summary(req: &mut dyn RequestExt) -> EndpointResult {
             .map(|((top_versions, krate), recent_downloads)| {
                 Ok(EncodableCrate::from_minimal(
                     krate,
-                    &top_versions,
+                    Some(&top_versions),
                     None,
                     false,
                     recent_downloads,
@@ -111,71 +112,183 @@ pub fn summary(req: &mut dyn RequestExt) -> EndpointResult {
 /// Handles the `GET /crates/:crate_id` route.
 pub fn show(req: &mut dyn RequestExt) -> EndpointResult {
     let name = &req.params()["crate_id"];
+    let include = req
+        .query()
+        .get("include")
+        .map(|mode| ShowIncludeMode::from_str(mode))
+        .transpose()?
+        .unwrap_or_default();
+
     let conn = req.db_read_only()?;
     let krate: Crate = Crate::by_name(name).first(&*conn)?;
 
-    let mut versions_and_publishers: Vec<(Version, Option<User>)> = krate
-        .all_versions()
-        .left_outer_join(users::table)
-        .select((versions::all_columns, users::all_columns.nullable()))
-        .load(&*conn)?;
-
-    versions_and_publishers
-        .sort_by_cached_key(|(version, _)| Reverse(semver::Version::parse(&version.num).ok()));
-
-    let versions = versions_and_publishers
-        .iter()
-        .map(|(v, _)| v)
-        .cloned()
-        .collect::<Vec<_>>();
-    let versions_publishers_and_audit_actions = versions_and_publishers
-        .into_iter()
-        .zip(VersionOwnerAction::for_versions(&conn, &versions)?.into_iter())
-        .map(|((v, pb), aas)| (v, pb, aas))
-        .collect::<Vec<_>>();
+    let versions_publishers_and_audit_actions = if include.versions {
+        let mut versions_and_publishers: Vec<(Version, Option<User>)> = krate
+            .all_versions()
+            .left_outer_join(users::table)
+            .select((versions::all_columns, users::all_columns.nullable()))
+            .load(&*conn)?;
+        versions_and_publishers
+            .sort_by_cached_key(|(version, _)| Reverse(semver::Version::parse(&version.num).ok()));
+
+        let versions = versions_and_publishers
+            .iter()
+            .map(|(v, _)| v)
+            .cloned()
+            .collect::<Vec<_>>();
+        Some(
+            versions_and_publishers
+                .into_iter()
+                .zip(VersionOwnerAction::for_versions(&conn, &versions)?.into_iter())
+                .map(|((v, pb), aas)| (v, pb, aas))
+                .collect::<Vec<_>>(),
+        )
+    } else {
+        None
+    };
     let ids = versions_publishers_and_audit_actions
-        .iter()
-        .map(|v| v.0.id)
-        .collect();
-
-    let kws = CrateKeyword::belonging_to(&krate)
-        .inner_join(keywords::table)
-        .select(keywords::all_columns)
-        .load(&*conn)?;
-    let cats = CrateCategory::belonging_to(&krate)
-        .inner_join(categories::table)
-        .select(categories::all_columns)
-        .load(&*conn)?;
-    let recent_downloads = RecentCrateDownloads::belonging_to(&krate)
-        .select(recent_crate_downloads::downloads)
-        .get_result(&*conn)
-        .optional()?;
+        .as_ref()
+        .map(|vps| vps.iter().map(|v| v.0.id).collect());
+
+    let kws = if include.keywords {
+        Some(
+            CrateKeyword::belonging_to(&krate)
+                .inner_join(keywords::table)
+                .select(keywords::all_columns)
+                .load(&*conn)?,
+        )
+    } else {
+        None
+    };
+    let cats = if include.categories {
+        Some(
+            CrateCategory::belonging_to(&krate)
+                .inner_join(categories::table)
+                .select(categories::all_columns)
+                .load(&*conn)?,
+        )
+    } else {
+        None
+    };
+    let recent_downloads = if include.downloads {
+        RecentCrateDownloads::belonging_to(&krate)
+            .select(recent_crate_downloads::downloads)
+            .get_result(&*conn)
+            .optional()?
+    } else {
+        None
+    };
 
-    let badges = badges::table
-        .filter(badges::crate_id.eq(krate.id))
-        .load(&*conn)?;
-    let top_versions = krate.top_versions(&conn)?;
+    let badges = if include.badges {
+        Some(
+            badges::table
+                .filter(badges::crate_id.eq(krate.id))
+                .load(&*conn)?,
+        )
+    } else {
+        None
+    };
+    let top_versions = if include.versions {
+        Some(krate.top_versions(&conn)?)
+    } else {
+        None
+    };
 
-    Ok(req.json(&json!({
-        "crate": EncodableCrate::from(
-            krate.clone(),
-            &top_versions,
-            Some(ids),
-            Some(&kws),
-            Some(&cats),
-            Some(badges),
-            false,
-            recent_downloads,
-        ),
-        "versions": versions_publishers_and_audit_actions
-            .into_iter()
+    let encodable_crate = EncodableCrate::from(
+        krate.clone(),
+        top_versions.as_ref(),
+        ids,
+        kws.as_deref(),
+        cats.as_deref(),
+        badges,
+        false,
+        recent_downloads,
+    );
+    let encodable_versions = versions_publishers_and_audit_actions.map(|vpa| {
+        vpa.into_iter()
             .map(|(v, pb, aas)| EncodableVersion::from(v, &krate.name, pb, aas))
-            .collect::<Vec<_>>(),
-        "keywords": kws.into_iter().map(Keyword::into).collect::<Vec<EncodableKeyword>>(),
-        "categories": cats.into_iter().map(Category::into).collect::<Vec<EncodableCategory>>(),
+            .collect::<Vec<_>>()
+    });
+    let encodable_keywords = kws.map(|kws| {
+        kws.into_iter()
+            .map(Keyword::into)
+            .collect::<Vec<EncodableKeyword>>()
+    });
+    let encodable_cats = cats.map(|cats| {
+        cats.into_iter()
+            .map(Category::into)
+            .collect::<Vec<EncodableCategory>>()
+    });
+    Ok(req.json(&json!({
+        "crate": encodable_crate,
+        "versions": encodable_versions,
+        "keywords": encodable_keywords,
+        "categories": encodable_cats,
     })))
 }
 
+#[derive(Debug)]
+struct ShowIncludeMode {
+    versions: bool,
+    keywords: bool,
+    categories: bool,
+    badges: bool,
+    downloads: bool,
+}
+
+impl Default for ShowIncludeMode {
+    fn default() -> Self {
+        // Send everything for legacy clients that expect the full response
+        Self {
+            versions: true,
+            keywords: true,
+            categories: true,
+            badges: true,
+            downloads: true,
+        }
+    }
+}
+
+impl ShowIncludeMode {
+    const INVALID_COMPONENT: &'static str =
+        "invalid component for ?include= (expected 'versions', 'keywords', 'categories', 'badges', 'downloads', or 'full')";
+}
+
+impl FromStr for ShowIncludeMode {
+    type Err = Box<dyn AppError>;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let mut mode = Self {
+            versions: false,
+            keywords: false,
+            categories: false,
+            badges: false,
+            downloads: false,
+        };
+        for component in s.split(',') {
+            match component {
+                "" => {}
+                "full" => {
+                    mode = Self {
+                        versions: true,
+                        keywords: true,
+                        categories: true,
+                        badges: true,
+                        downloads: true,
+                    }
+                }
+                "versions" => mode.versions = true,
+                "keywords" => mode.keywords = true,
+                "categories" => mode.categories = true,
+                "badges" => mode.badges = true,
+                "downloads" => mode.downloads = true,
+                _ => return Err(bad_request(Self::INVALID_COMPONENT)),
+            }
+        }
+        Ok(mode)
+    }
+}
+
 /// Handles the `GET /crates/:crate_id/:version/readme` route.
 pub fn readme(req: &mut dyn RequestExt) -> EndpointResult {
     let crate_name = &req.params()["crate_id"];

src/controllers/krate/publish.rs

@@ -253,7 +253,7 @@ pub fn publish(req: &mut dyn RequestExt) -> EndpointResult {
         };
 
         Ok(req.json(&GoodCrate {
-            krate: EncodableCrate::from_minimal(krate, &top_versions, None, false, None),
+            krate: EncodableCrate::from_minimal(krate, Some(&top_versions), None, false, None),
             warnings,
         }))
     })

src/controllers/krate/search.rs

@@ -335,7 +335,7 @@ pub fn search(req: &mut dyn RequestExt) -> EndpointResult {
             |((((max_version, krate), perfect_match), recent_downloads), badges)| {
                 EncodableCrate::from_minimal(
                     krate,
-                    &max_version,
+                    Some(&max_version),
                     Some(badges),
                     perfect_match,
                     Some(recent_downloads),

src/tests/all.rs

@@ -69,8 +69,8 @@ struct CrateMeta {
 pub struct CrateResponse {
     #[serde(rename = "crate")]
     krate: EncodableCrate,
-    versions: Vec<EncodableVersion>,
-    keywords: Vec<EncodableKeyword>,
+    versions: Option<Vec<EncodableVersion>>,
+    keywords: Option<Vec<EncodableKeyword>>,
 }
 #[derive(Deserialize)]
 pub struct VersionResponse {

src/tests/krate/show.rs

@@ -43,31 +43,79 @@ fn show() {
     assert_eq!(json.krate.documentation, krate.documentation);
     assert_eq!(json.krate.keywords, Some(vec!["kw1".into()]));
     assert_eq!(json.krate.recent_downloads, Some(10));
-    let versions = json.krate.versions.as_ref().unwrap();
+    let crate_versions = json.krate.versions.as_ref().unwrap();
+    assert_eq!(crate_versions.len(), 3);
+    let versions = json.versions.as_ref().unwrap();
     assert_eq!(versions.len(), 3);
-    assert_eq!(json.versions.len(), 3);
 
-    assert_eq!(json.versions[0].id, versions[0]);
-    assert_eq!(json.versions[0].krate, json.krate.id);
-    assert_eq!(json.versions[0].num, "1.0.0");
-    assert_none!(&json.versions[0].published_by);
+    assert_eq!(versions[0].id, crate_versions[0]);
+    assert_eq!(versions[0].krate, json.krate.id);
+    assert_eq!(versions[0].num, "1.0.0");
+    assert_none!(&versions[0].published_by);
     let suffix = "/api/v1/crates/foo_show/1.0.0/download";
     assert!(
-        json.versions[0].dl_path.ends_with(suffix),
+        versions[0].dl_path.ends_with(suffix),
         "bad suffix {}",
-        json.versions[0].dl_path
+        versions[0].dl_path
     );
-    assert_eq!(1, json.keywords.len());
-    assert_eq!("kw1", json.keywords[0].id);
+    let keywords = json.keywords.as_ref().unwrap();
+    assert_eq!(1, keywords.len());
+    assert_eq!("kw1", keywords[0].id);
 
-    assert_eq!(json.versions[1].num, "0.5.1");
-    assert_eq!(json.versions[2].num, "0.5.0");
+    assert_eq!(versions[1].num, "0.5.1");
+    assert_eq!(versions[2].num, "0.5.0");
     assert_eq!(
-        json.versions[1].published_by.as_ref().unwrap().login,
+        versions[1].published_by.as_ref().unwrap().login,
         user.gh_login
     );
 }
 
+#[test]
+fn show_minimal() {
+    let (app, anon, user) = TestApp::init().with_user();
+    let user = user.as_model();
+
+    let krate = app.db(|conn| {
+        use cargo_registry::schema::versions;
+        use diesel::{update, ExpressionMethods};
+
+        let krate = CrateBuilder::new("foo_show_minimal", user.id)
+            .description("description")
+            .documentation("https://example.com")
+            .homepage("http://example.com")
+            .version(VersionBuilder::new("1.0.0"))
+            .version(VersionBuilder::new("0.5.0"))
+            .version(VersionBuilder::new("0.5.1"))
+            .keyword("kw1")
+            .downloads(20)
+            .recent_downloads(10)
+            .expect_build(conn);
+
+        // Make version 1.0.0 mimic a version published before we started recording who published
+        // versions
+        let none: Option<i32> = None;
+        update(versions::table)
+            .filter(versions::num.eq("1.0.0"))
+            .set(versions::published_by.eq(none))
+            .execute(conn)
+            .unwrap();
+
+        krate
+    });
+
+    let json = anon.show_crate_minimal("foo_show_minimal");
+    assert_eq!(json.krate.name, krate.name);
+    assert_eq!(json.krate.id, krate.name);
+    assert_eq!(json.krate.description, krate.description);
+    assert_eq!(json.krate.homepage, krate.homepage);
+    assert_eq!(json.krate.documentation, krate.documentation);
+    assert_eq!(json.krate.keywords, None);
+    assert_eq!(json.krate.recent_downloads, None);
+    assert_eq!(json.krate.versions, None);
+    assert!(json.versions.is_none());
+    assert!(json.keywords.is_none());
+}
+
 #[test]
 fn block_bad_documentation_url() {
     let (app, anon, user) = TestApp::init().with_user();