From 89be03b452e0fffdd7d800067ea7bf7b1f39321c Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 14 Nov 2023 21:03:22 +0800 Subject: [PATCH] feat: cache per page --- app/api/github.ts | 139 +++++++++++++++++++++++++++------------------- types/index.ts | 5 -- 2 files changed, 81 insertions(+), 63 deletions(-) diff --git a/app/api/github.ts b/app/api/github.ts index cb47c16..6327b50 100644 --- a/app/api/github.ts +++ b/app/api/github.ts @@ -1,6 +1,5 @@ import { LRUCache } from "lru-cache" import { GhUser, GhUserUse } from "./types" -import { UsedRepoInfo } from "@/types" import fs from "node:fs/promises" import path from "node:path" import { throttle } from "@/utils" @@ -10,35 +9,56 @@ const repoCache = new LRUCache({ ttl: 1000 * 60 * 60 * 24, }) -const cache_file_path = "./data/cache.json" +const repo_cache_file_path = "./data/cache.json" +const repo_exists_file_path = "./data/repo_exists.json" -async function saveCache() { +async function loadByPath( + cache: LRUCache, + filePath: string +) { try { - console.log("--- saving cache ---") - const items = repoCache.dump() - await fs.writeFile(cache_file_path, JSON.stringify(items)) - console.log("--- cache saved ---") + const items = JSON.parse(await fs.readFile(filePath, "utf-8")) + cache.load(items) + console.log(`--- ${filePath} cache loaded ---`) } catch (e) { - console.log("--- failed to save cache ---", e) + console.log(`--- ${filePath} no cache found ---`, e) + } +} + +async function saveByPath( + cache: LRUCache, + filePath: string +) { + try { + console.log(`--- ${filePath} saving cache ---`) + const items = cache.dump() + await fs.writeFile(filePath, JSON.stringify(items)) + console.log(`--- ${filePath} cache saved ---`) + } catch (e) { + console.log(`--- ${filePath} failed to save cache ---`, e) } } async function loadCache() { console.log(`--- PAT: ${process.env.PAT} ---`) try { - await fs.mkdir(path.dirname(cache_file_path), { recursive: true }) - const items = JSON.parse(await fs.readFile(cache_file_path, "utf-8")) - repoCache.load(items) - console.log("--- cache loaded ---") + await fs.mkdir(path.dirname(repo_cache_file_path), { recursive: true }) + loadByPath(repoCache, repo_cache_file_path) + loadByPath(repoExists, repo_exists_file_path) } catch (e) { - console.log("--- no cache found ---", e) + console.log(e) } } loadCache() -const throttleSaveCache = throttle(saveCache, 1000 * 60) +const throttleSaveRepoCache = throttle(() => { + saveByPath(repoCache, repo_cache_file_path) +}, 1000 * 60) +const throttleSaveRepoExists = throttle(() => { + saveByPath(repoExists, repo_exists_file_path) +}, 1000 * 60) -const repoNotFoundCache = new LRUCache({ +const repoExists = new LRUCache({ max: 100, ttl: 1000 * 60 * 60 * 12, }) @@ -49,36 +69,24 @@ const avatarCache = new LRUCache({ }) export function usedBy(per_page: number, page: number) { - let all = [] as UsedRepoInfo[] - repoCache.forEach((value, key) => { - all.push({ - name: key, - count: value.length, - }) + let repos = [] as string[] + repoExists.forEach((value, key) => { + if (value) { + repos.push(key) + } }) - all = all.sort((a, b) => b.count - a.count) - const res = all.slice((page - 1) * per_page, page * per_page) + const res = repos.slice((page - 1) * per_page, page * per_page) return { data: res, - total: all.length, } } -export async function fetchRepo(repo: string, maxPages: number = 1) { - // validate repo - const repoRegex = /^[\w-]+\/[\w-]+$/ - if (!repoRegex.test(repo)) { - throw new Error(`invalid repo: ${repo}`) - } - if (repoCache.has(repo)) { - return repoCache.get(repo)! - } - if (repoNotFoundCache.has(repo)) { - throw new Error(`repo ${repo} not found`) +async function fetchRepoOnePage(repo: string, page: number) { + const cacheKey = `${repo}-${page}` + if (repoCache.has(cacheKey)) { + return repoCache.get(cacheKey)! } - console.log(`fetching ${repo}`) - const users = [] - let page = 1 + console.log(`fetching ${cacheKey}`) let fetchInit: Parameters[1] if (process.env.PAT) { fetchInit = { @@ -87,26 +95,19 @@ export async function fetchRepo(repo: string, maxPages: number = 1) { }, } } - while (page <= maxPages) { - const res = await fetch( - `https://api.github.com/repos/${repo}/contributors?per_page=100&page=${page}`, - fetchInit - ) - const usersPage = await res.json() - if (usersPage.message) { - if (usersPage.message === "Not Found") { - repoNotFoundCache.set(repo, true) - throw new Error(`repo ${repo} not found`) - } - throw new Error(`failed to fetch repo ${repo}: ${usersPage.message}`) - } - if (usersPage.length === 0) { - break + const res = await fetch( + `https://api.github.com/repos/${repo}/contributors?per_page=100&page=${page}`, + fetchInit + ) + const usersPage = await res.json() + if (usersPage.message) { + if (usersPage.message === "Not Found") { + repoExists.set(repo, false) + throw new Error(`repo ${repo} not found`) } - users.push(...usersPage) - page++ + throw new Error(`failed to fetch repo ${cacheKey}: ${usersPage.message}`) } - const usersUse = (users as GhUser[]).map((user): GhUserUse => { + const usersUse = (usersPage as GhUser[]).map((user): GhUserUse => { return { login: user.login, avatar_url: user.avatar_url, @@ -114,11 +115,33 @@ export async function fetchRepo(repo: string, maxPages: number = 1) { contributions: user.contributions, } }) - repoCache.set(repo, usersUse) - throttleSaveCache() + repoCache.set(cacheKey, usersUse) + throttleSaveRepoCache() + repoExists.set(repo, true) + throttleSaveRepoExists() return usersUse } +export async function fetchRepo(repo: string, maxPages: number = 1) { + // validate repo + const repoRegex = /^[\w-]+\/[\w-]+$/ + if (!repoRegex.test(repo)) { + throw new Error(`invalid repo: ${repo}`) + } + if (repoExists.get(repo) === false) { + throw new Error(`repo ${repo} not found`) + } + console.log(`fetching ${repo}`) + const users = [] as GhUserUse[] + let page = 1 + while (page <= maxPages) { + const usersPage = await fetchRepoOnePage(repo, page) + users.push(...usersPage) + page++ + } + return users +} + export async function fetchRepos(repos: string[], maxPages?: number) { const users = ( await Promise.all( diff --git a/types/index.ts b/types/index.ts index 182907f..61c2de9 100644 --- a/types/index.ts +++ b/types/index.ts @@ -3,8 +3,3 @@ import { SVGProps } from "react" export type IconSvgProps = SVGProps & { size?: number } - -export type UsedRepoInfo = { - name: string - count: number -}