feat: cache per page

xhofe · Nov 14, 2023 · 89be03b · 89be03b
1 parent 068455b
commit 89be03b
Show file tree

Hide file tree

Showing 2 changed files with 81 additions and 63 deletions.
diff --git a/app/api/github.ts b/app/api/github.ts
@@ -1,6 +1,5 @@
 import { LRUCache } from "lru-cache"
 import { GhUser, GhUserUse } from "./types"
-import { UsedRepoInfo } from "@/types"
 import fs from "node:fs/promises"
 import path from "node:path"
 import { throttle } from "@/utils"
@@ -10,35 +9,56 @@ const repoCache = new LRUCache<string, GhUserUse[]>({
   ttl: 1000 * 60 * 60 * 24,
 })
 
-const cache_file_path = "./data/cache.json"
+const repo_cache_file_path = "./data/cache.json"
+const repo_exists_file_path = "./data/repo_exists.json"
 
-async function saveCache() {
+async function loadByPath<K extends {}, V extends {}>(
+  cache: LRUCache<K, V>,
+  filePath: string
+) {
   try {
-    console.log("--- saving cache ---")
-    const items = repoCache.dump()
-    await fs.writeFile(cache_file_path, JSON.stringify(items))
-    console.log("--- cache saved ---")
+    const items = JSON.parse(await fs.readFile(filePath, "utf-8"))
+    cache.load(items)
+    console.log(`--- ${filePath} cache loaded ---`)
   } catch (e) {
-    console.log("--- failed to save cache ---", e)
+    console.log(`--- ${filePath} no cache found ---`, e)
+  }
+}
+
+async function saveByPath<K extends {}, V extends {}>(
+  cache: LRUCache<K, V>,
+  filePath: string
+) {
+  try {
+    console.log(`--- ${filePath} saving cache ---`)
+    const items = cache.dump()
+    await fs.writeFile(filePath, JSON.stringify(items))
+    console.log(`--- ${filePath} cache saved ---`)
+  } catch (e) {
+    console.log(`--- ${filePath} failed to save cache ---`, e)
   }
 }
 
 async function loadCache() {
   console.log(`--- PAT: ${process.env.PAT} ---`)
   try {
-    await fs.mkdir(path.dirname(cache_file_path), { recursive: true })
-    const items = JSON.parse(await fs.readFile(cache_file_path, "utf-8"))
-    repoCache.load(items)
-    console.log("--- cache loaded ---")
+    await fs.mkdir(path.dirname(repo_cache_file_path), { recursive: true })
+    loadByPath(repoCache, repo_cache_file_path)
+    loadByPath(repoExists, repo_exists_file_path)
   } catch (e) {
-    console.log("--- no cache found ---", e)
+    console.log(e)
   }
 }
 
 loadCache()
-const throttleSaveCache = throttle(saveCache, 1000 * 60)
+const throttleSaveRepoCache = throttle(() => {
+  saveByPath(repoCache, repo_cache_file_path)
+}, 1000 * 60)
+const throttleSaveRepoExists = throttle(() => {
+  saveByPath(repoExists, repo_exists_file_path)
+}, 1000 * 60)
 
-const repoNotFoundCache = new LRUCache<string, boolean>({
+const repoExists = new LRUCache<string, boolean>({
   max: 100,
   ttl: 1000 * 60 * 60 * 12,
 })
@@ -49,36 +69,24 @@ const avatarCache = new LRUCache<string, Buffer>({
 })
 
 export function usedBy(per_page: number, page: number) {
-  let all = [] as UsedRepoInfo[]
-  repoCache.forEach((value, key) => {
-    all.push({
-      name: key,
-      count: value.length,
-    })
+  let repos = [] as string[]
+  repoExists.forEach((value, key) => {
+    if (value) {
+      repos.push(key)
+    }
   })
-  all = all.sort((a, b) => b.count - a.count)
-  const res = all.slice((page - 1) * per_page, page * per_page)
+  const res = repos.slice((page - 1) * per_page, page * per_page)
   return {
     data: res,
-    total: all.length,
   }
 }
 
-export async function fetchRepo(repo: string, maxPages: number = 1) {
-  // validate repo
-  const repoRegex = /^[\w-]+\/[\w-]+$/
-  if (!repoRegex.test(repo)) {
-    throw new Error(`invalid repo: ${repo}`)
-  }
-  if (repoCache.has(repo)) {
-    return repoCache.get(repo)!
-  }
-  if (repoNotFoundCache.has(repo)) {
-    throw new Error(`repo ${repo} not found`)
+async function fetchRepoOnePage(repo: string, page: number) {
+  const cacheKey = `${repo}-${page}`
+  if (repoCache.has(cacheKey)) {
+    return repoCache.get(cacheKey)!
   }
-  console.log(`fetching ${repo}`)
-  const users = []
-  let page = 1
+  console.log(`fetching ${cacheKey}`)
   let fetchInit: Parameters<typeof fetch>[1]
   if (process.env.PAT) {
     fetchInit = {
@@ -87,38 +95,53 @@ export async function fetchRepo(repo: string, maxPages: number = 1) {
       },
     }
   }
-  while (page <= maxPages) {
-    const res = await fetch(
-      `https://api.github.com/repos/${repo}/contributors?per_page=100&page=${page}`,
-      fetchInit
-    )
-    const usersPage = await res.json()
-    if (usersPage.message) {
-      if (usersPage.message === "Not Found") {
-        repoNotFoundCache.set(repo, true)
-        throw new Error(`repo ${repo} not found`)
-      }
-      throw new Error(`failed to fetch repo ${repo}: ${usersPage.message}`)
-    }
-    if (usersPage.length === 0) {
-      break
+  const res = await fetch(
+    `https://api.github.com/repos/${repo}/contributors?per_page=100&page=${page}`,
+    fetchInit
+  )
+  const usersPage = await res.json()
+  if (usersPage.message) {
+    if (usersPage.message === "Not Found") {
+      repoExists.set(repo, false)
+      throw new Error(`repo ${repo} not found`)
     }
-    users.push(...usersPage)
-    page++
+    throw new Error(`failed to fetch repo ${cacheKey}: ${usersPage.message}`)
   }
-  const usersUse = (users as GhUser[]).map((user): GhUserUse => {
+  const usersUse = (usersPage as GhUser[]).map((user): GhUserUse => {
     return {
       login: user.login,
       avatar_url: user.avatar_url,
       type: user.type,
       contributions: user.contributions,
     }
   })
-  repoCache.set(repo, usersUse)
-  throttleSaveCache()
+  repoCache.set(cacheKey, usersUse)
+  throttleSaveRepoCache()
+  repoExists.set(repo, true)
+  throttleSaveRepoExists()
   return usersUse
 }
 
+export async function fetchRepo(repo: string, maxPages: number = 1) {
+  // validate repo
+  const repoRegex = /^[\w-]+\/[\w-]+$/
+  if (!repoRegex.test(repo)) {
+    throw new Error(`invalid repo: ${repo}`)
+  }
+  if (repoExists.get(repo) === false) {
+    throw new Error(`repo ${repo} not found`)
+  }
+  console.log(`fetching ${repo}`)
+  const users = [] as GhUserUse[]
+  let page = 1
+  while (page <= maxPages) {
+    const usersPage = await fetchRepoOnePage(repo, page)
+    users.push(...usersPage)
+    page++
+  }
+  return users
+}
+
 export async function fetchRepos(repos: string[], maxPages?: number) {
   const users = (
     await Promise.all(

diff --git a/types/index.ts b/types/index.ts
@@ -3,8 +3,3 @@ import { SVGProps } from "react"
 export type IconSvgProps = SVGProps<SVGSVGElement> & {
   size?: number
 }
-
-export type UsedRepoInfo = {
-  name: string
-  count: number
-}