feat: organizations support #2

readme.md

@@ -50,6 +50,12 @@ services:
 4. Click genereate token & copy it
 5. Save token to `.env` file with name `GITHUB_TOKEN=???`
 
+## How it works?
+
+Every hour `ghstats` loads the list of public repositories and their statistics, and saves the data in SQLite. If at the first startup there is no repositories in the database, synchronization will happen immediately, if `ghstats` is restarted again, synchronization will be performed according to the scheduler. Data is stored per day, re-fetching data for the current day will update existing records in the database.
+
+All public repositories that can be accessed are saved. If you need more detailed configuration – open PR please.
+
 ## 🤝 Contributing
 
 All contributions are welcome! Feel free to open an issue or submit a pull request.

src/db_client.rs

@@ -297,7 +297,7 @@ impl DbClient {
   }
 
   pub async fn insert_stats(&self, repo: &Repo, date: &str) -> Res {
-    self.insert_repo(repo).await?;
+    let _ = self.insert_repo(repo).await?;
 
     let qs = "
     INSERT INTO repo_stats AS t (repo_id, date, stars, forks, watchers, issues)

src/gh_client.rs

@@ -86,7 +86,8 @@ impl GhClient {
     let mut page = 1;
 
     loop {
-      let url = format!("{}/user/repos?type=owner&per_page=100&page={}", self.base_url, page);
+      let url = format!("{}/user/repos?visibility=public", self.base_url);
+      let url = format!("{}&per_page=100&page={}", url, page);
       let rep = self.client.get(url).send().await?.error_for_status()?;
 
       let link = match rep.headers().get("link") {

src/main.rs

@@ -9,7 +9,7 @@ mod pages;
 mod utils;
 
 use db_client::{DbClient, RepoFilter};
-use gh_client::GhClient;
+use gh_client::{GhClient, Repo};
 use utils::Res;
 
 struct AppState {
@@ -41,6 +41,21 @@ async fn health() -> impl IntoResponse {
   (StatusCode::OK, axum::response::Json(msg))
 }
 
+async fn update_repo_metrics(db: &DbClient, gh: &GhClient, repo: &Repo, date: &str) -> Res {
+  let views = gh.traffic_views(&repo.full_name).await?;
+  let clones = gh.traffic_clones(&repo.full_name).await?;
+  let referrers = gh.traffic_refs(&repo.full_name).await?;
+  let popular_paths = gh.traffic_paths(&repo.full_name).await?;
+
+  db.insert_stats(&repo, date).await?;
+  db.insert_views(&repo, &views).await?;
+  db.insert_clones(&repo, &clones).await?;
+  db.insert_referrers(&repo, date, &referrers).await?;
+  db.insert_paths(&repo, date, &popular_paths).await?;
+
+  Ok(())
+}
+
 async fn update_metrics(db: &DbClient, gh: &GhClient) -> Res {
   let stime = std::time::Instant::now();
 
@@ -49,16 +64,14 @@ async fn update_metrics(db: &DbClient, gh: &GhClient) -> Res {
 
   let repos = gh.get_repos().await?;
   for repo in repos {
-    let views = gh.traffic_views(&repo.full_name).await?;
-    let clones = gh.traffic_clones(&repo.full_name).await?;
-    let referrers = gh.traffic_refs(&repo.full_name).await?;
-    let popular_paths = gh.traffic_paths(&repo.full_name).await?;
-
-    db.insert_stats(&repo, &date).await?;
-    db.insert_views(&repo, &views).await?;
-    db.insert_clones(&repo, &clones).await?;
-    db.insert_referrers(&repo, &date, &referrers).await?;
-    db.insert_paths(&repo, &date, &popular_paths).await?;
+    match update_repo_metrics(db, gh, &repo, &date).await {
+      Err(e) => {
+        tracing::warn!("error updating metrics for {}: {:?}", repo.full_name, e);
+        continue;
+      }
+      // Ok(_) => tracing::info!("updated metrics for {}", repo.full_name),
+      Ok(_) => {}
+    }
   }
 
   tracing::info!("update_metrics took {:?}", stime.elapsed());