Auto merge of rust-lang#10466 - samueltardieu:popular-crates, r=llogiq

Add the `popular-crates` binary

This program downloads crates info from <https://crates.io/> and builds a TOML file that can be fed to `lintcheck`.

I have been asked, on various pull requests, what the result of `lintcheck` was. However, the default configuration file for lintcheck is limited. This `popular-crates` program allows building a recent list of the recently most downloaded crates from <https://crates.io> and feed it to `lintcheck`. Using it, it was easy to test two new lints against the 500 recently most downloaded crates to ensure that there was no regression.

changelog: none

lintcheck/Cargo.toml

@@ -8,12 +8,16 @@ repository = "https://github.com/rust-lang/rust-clippy"
 categories = ["development-tools"]
 edition = "2021"
 publish = false
+default-run = "lintcheck"
 
 [dependencies]
+anyhow = "1.0.69"
 cargo_metadata = "0.15.3"
 clap = { version = "4.1.8", features = ["derive", "env"] }
+crates_io_api = "0.8.1"
 crossbeam-channel = "0.5.6"
 flate2 = "1.0"
+indicatif = "0.17.3"
 rayon = "1.5.1"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.85"
@@ -24,3 +28,11 @@ walkdir = "2.3"
 
 [features]
 deny-warnings = []
+
+[[bin]]
+name = "lintcheck"
+path = "src/main.rs"
+
+[[bin]]
+name = "popular-crates"
+path = "src/popular-crates.rs"

lintcheck/README.md

@@ -25,6 +25,15 @@ the repo root.
 
 The results will then be saved to `lintcheck-logs/custom_logs.toml`.
 
+The `custom.toml` file may be built using <https://crates.io> recently most
+downloaded crates by using the `popular-crates` binary from the `lintcheck`
+directory. For example, to retrieve the 100 recently most downloaded crates:
+
+```
+cargo run --release --bin popular-crates -- -n 100 custom.toml
+```
+
+
 ### Configuring the Crate Sources
 
 The sources to check are saved in a `toml` file. There are three types of

lintcheck/src/popular-crates.rs

@@ -0,0 +1,65 @@
+#![deny(clippy::pedantic)]
+
+use clap::Parser;
+use crates_io_api::{CratesQueryBuilder, Sort, SyncClient};
+use indicatif::ProgressBar;
+use std::collections::HashSet;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+use std::path::PathBuf;
+use std::time::Duration;
+
+#[derive(Parser)]
+struct Opts {
+    /// Output TOML file name
+    output: PathBuf,
+    /// Number of crate names to download
+    #[clap(short, long, default_value_t = 100)]
+    number: usize,
+    /// Do not output progress
+    #[clap(short, long)]
+    quiet: bool,
+}
+
+fn main() -> anyhow::Result<()> {
+    let opts = Opts::parse();
+    let mut output = BufWriter::new(File::create(opts.output)?);
+    output.write_all(b"[crates]\n")?;
+    let client = SyncClient::new(
+        "clippy/lintcheck (github.com/rust-lang/rust-clippy/)",
+        Duration::from_secs(1),
+    )?;
+    let mut seen_crates = HashSet::new();
+    let pb = if opts.quiet {
+        None
+    } else {
+        Some(ProgressBar::new(opts.number as u64))
+    };
+    let mut query = CratesQueryBuilder::new()
+        .sort(Sort::RecentDownloads)
+        .page_size(100)
+        .build();
+    while seen_crates.len() < opts.number {
+        let retrieved = client.crates(query.clone())?.crates;
+        if retrieved.is_empty() {
+            eprintln!("No more than {} crates available from API", seen_crates.len());
+            break;
+        }
+        for c in retrieved {
+            if seen_crates.insert(c.name.clone()) {
+                output.write_all(
+                    format!(
+                        "{} = {{ name = '{}', versions = ['{}'] }}\n",
+                        c.name, c.name, c.max_version
+                    )
+                    .as_bytes(),
+                )?;
+                if let Some(pb) = &pb {
+                    pb.inc(1);
+                }
+            }
+        }
+        query.set_page(query.page() + 1);
+    }
+    Ok(())
+}