Graceful error handling when calling code host apis

CHANGELOG.md

@@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added config option `settings.reindexInterval` and `settings.resyncInterval` to control how often the index should be re-indexed and re-synced. ([#134](https://github.com/sourcebot-dev/sourcebot/pull/134))
 - Added `exclude.size` to the GitHub config to allow excluding repositories by size. ([#137](https://github.com/sourcebot-dev/sourcebot/pull/137))
 
+### Fixed
+
+- Fixed issue where config synchronization was failing entirely when a single api call fails. ([#142](https://github.com/sourcebot-dev/sourcebot/pull/142))
+
 ## [2.6.2] - 2024-12-13
 
 ### Added

packages/backend/src/gerrit.ts

@@ -28,9 +28,18 @@ export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppCon
    const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
    const hostname = new URL(config.url).hostname;
 
-   const { durationMs, data: projects } = await measure(() =>
-      fetchAllProjects(url)
-   );
+   const { durationMs, data: projects } = await measure(async () => {
+      try {
+         return fetchAllProjects(url)
+      } catch (err) {
+         logger.error(`Failed to fetch projects from ${url}`, err);
+         return null;
+      }
+   });
+
+   if (!projects) {
+      return [];
+   }
 
    // exclude "All-Projects" and "All-Users" projects
    delete projects['All-Projects'];

packages/backend/src/gitea.ts

@@ -122,73 +122,98 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, ctx: AppConte
 }
 
 const getTagsForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
-    logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
-    const { durationMs, data: tags } = await measure(() =>
-        paginate((page) => api.repos.repoListTags(owner, repo, {
-            page
-        }))
-    );
-    logger.debug(`Found ${tags.length} tags in repo ${owner}/${repo} in ${durationMs}ms.`);
-    return tags;
+    try {
+        logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
+        const { durationMs, data: tags } = await measure(() =>
+            paginate((page) => api.repos.repoListTags(owner, repo, {
+                page
+            }))
+        );
+        logger.debug(`Found ${tags.length} tags in repo ${owner}/${repo} in ${durationMs}ms.`);
+        return tags;
+    } catch (e) {
+        logger.error(`Failed to fetch tags for repo ${owner}/${repo}.`, e);
+        return [];
+    }
 }
 
 const getBranchesForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
-    logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
-    const { durationMs, data: branches } = await measure(() => 
-        paginate((page) => api.repos.repoListBranches(owner, repo, {
-            page
-        }))
-    );
-    logger.debug(`Found ${branches.length} branches in repo ${owner}/${repo} in ${durationMs}ms.`);
-    return branches;
+    try {
+        logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
+        const { durationMs, data: branches } = await measure(() => 
+            paginate((page) => api.repos.repoListBranches(owner, repo, {
+                page
+            }))
+        );
+        logger.debug(`Found ${branches.length} branches in repo ${owner}/${repo} in ${durationMs}ms.`);
+        return branches;
+    } catch (e) {
+        logger.error(`Failed to fetch branches for repo ${owner}/${repo}.`, e);
+        return [];
+    }
 }
 
 const getReposOwnedByUsers = async <T>(users: string[], api: Api<T>) => {
     const repos = (await Promise.all(users.map(async (user) => {
-        logger.debug(`Fetching repos for user ${user}...`);
-
-        const { durationMs, data } = await measure(() =>
-            paginate((page) => api.users.userListRepos(user, {
-                page,
-            }))
-        );
-
-        logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`);
-        return data;
+        try {
+            logger.debug(`Fetching repos for user ${user}...`);
+
+            const { durationMs, data } = await measure(() =>
+                paginate((page) => api.users.userListRepos(user, {
+                    page,
+                }))
+            );
+
+            logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`);
+            return data;
+        } catch (e) {
+            logger.error(`Failed to fetch repos for user ${user}.`, e);
+            return [];
+        }
     }))).flat();
 
     return repos;
 }
 
 const getReposForOrgs = async <T>(orgs: string[], api: Api<T>) => {
     return (await Promise.all(orgs.map(async (org) => {
-        logger.debug(`Fetching repos for org ${org}...`);
-
-        const { durationMs, data } = await measure(() =>
-            paginate((page) => api.orgs.orgListRepos(org, {
-                limit: 100,
-                page,
-            }))
-        );
-
-        logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`);
-        return data;
+        try {
+            logger.debug(`Fetching repos for org ${org}...`);
+
+            const { durationMs, data } = await measure(() =>
+                paginate((page) => api.orgs.orgListRepos(org, {
+                    limit: 100,
+                    page,
+                }))
+            );
+
+            logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`);
+            return data;
+        } catch (e) {
+            logger.error(`Failed to fetch repos for org ${org}.`, e);
+            return [];
+        }
     }))).flat();
 }
 
 const getRepos = async <T>(repos: string[], api: Api<T>) => {
-    return Promise.all(repos.map(async (repo) => {
-        logger.debug(`Fetching repository info for ${repo}...`);
+    return (await Promise.all(repos.map(async (repo) => {
+        try {
+            logger.debug(`Fetching repository info for ${repo}...`);
 
-        const [owner, repoName] = repo.split('/');
-        const { durationMs, data: response } = await measure(() =>
-            api.repos.repoGet(owner, repoName),
-        );
+            const [owner, repoName] = repo.split('/');
+            const { durationMs, data: response } = await measure(() =>
+                api.repos.repoGet(owner, repoName),
+            );
 
-        logger.debug(`Found repo ${repo} in ${durationMs}ms.`);
+            logger.debug(`Found repo ${repo} in ${durationMs}ms.`);
 
-        return response.data;
-    }));
+            return [response.data];
+        } catch (e) {
+            logger.error(`Failed to fetch repository info for ${repo}.`, e);
+            return [];
+        }
+    }))).flat();
 }
 
 // @see : https://docs.gitea.com/development/api-usage#pagination

packages/backend/src/github.ts

@@ -201,114 +201,129 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
 }
 
 const getTagsForRepo = async (owner: string, repo: string, octokit: Octokit, signal: AbortSignal) => {
-    logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
-
-    const { durationMs, data: tags } = await measure(() => octokit.paginate(octokit.repos.listTags, {
-        owner,
-        repo,
-        per_page: 100,
-        request: {
-            signal
-        }
-    }));
+    try {
+        logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
+        const { durationMs, data: tags } = await measure(() => octokit.paginate(octokit.repos.listTags, {
+            owner,
+            repo,
+            per_page: 100,
+            request: {
+                signal
+            }
+        }));
 
-    logger.debug(`Found ${tags.length} tags for repo ${owner}/${repo} in ${durationMs}ms`);
-    return tags;
+        logger.debug(`Found ${tags.length} tags for repo ${owner}/${repo} in ${durationMs}ms`);
+        return tags;
+    } catch (e) {
+        logger.debug(`Error fetching tags for repo ${owner}/${repo}: ${e}`);
+        return [];
+    }
 }
 
 const getBranchesForRepo = async (owner: string, repo: string, octokit: Octokit, signal: AbortSignal) => {
-    logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
-    const { durationMs, data: branches } = await measure(() => octokit.paginate(octokit.repos.listBranches, {
-        owner,
-        repo,
-        per_page: 100,
-        request: {
-            signal
-        }
-    }));
-    logger.debug(`Found ${branches.length} branches for repo ${owner}/${repo} in ${durationMs}ms`);
-    return branches;
+    try {
+        logger.debug(`Fetching branches for repo ${owner}/${repo}...`);
+        const { durationMs, data: branches } = await measure(() => octokit.paginate(octokit.repos.listBranches, {
+            owner,
+            repo,
+            per_page: 100,
+            request: {
+                signal
+            }
+        }));
+        logger.debug(`Found ${branches.length} branches for repo ${owner}/${repo} in ${durationMs}ms`);
+        return branches;
+    } catch (e) {
+        logger.debug(`Error fetching branches for repo ${owner}/${repo}: ${e}`);
+        return [];
+    }
 }
 
 
 const getReposOwnedByUsers = async (users: string[], isAuthenticated: boolean, octokit: Octokit, signal: AbortSignal) => {
-    // @todo : error handling
     const repos = (await Promise.all(users.map(async (user) => {
-        logger.debug(`Fetching repository info for user ${user}...`);
-        const start = Date.now();
-
-        const result = await (() => {
-            if (isAuthenticated) {
-                return octokit.paginate(octokit.repos.listForAuthenticatedUser, {
-                    username: user,
-                    visibility: 'all',
-                    affiliation: 'owner',
-                    per_page: 100,
-                    request: {
-                        signal,
-                    },
-                });
-            } else {
-                return octokit.paginate(octokit.repos.listForUser, {
-                    username: user,
-                    per_page: 100,
-                    request: {
-                        signal,
-                    },
-                });
-            }
-        })();
-
-        const duration = Date.now() - start;
-        logger.debug(`Found ${result.length} owned by user ${user} in ${duration}ms.`);
-
-        return result;
+        try {
+            logger.debug(`Fetching repository info for user ${user}...`);
+
+            const { durationMs, data } = await measure(async () => {
+                if (isAuthenticated) {
+                    return octokit.paginate(octokit.repos.listForAuthenticatedUser, {
+                        username: user,
+                        visibility: 'all',
+                        affiliation: 'owner',
+                        per_page: 100,
+                        request: {
+                            signal,
+                        },
+                    });
+                } else {
+                    return octokit.paginate(octokit.repos.listForUser, {
+                        username: user,
+                        per_page: 100,
+                        request: {
+                            signal,
+                        },
+                    });
+                }
+            });
+
+            logger.debug(`Found ${data.length} owned by user ${user} in ${durationMs}ms.`);
+            return data;
+        } catch (e) {
+            logger.error(`Failed to fetch repository info for user ${user}.`, e);
+            return [];
+        }
     }))).flat();
 
     return repos;
 }
 
 const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal) => {
     const repos = (await Promise.all(orgs.map(async (org) => {
-        logger.debug(`Fetching repository info for org ${org}...`);
-        const start = Date.now();
-
-        const result = await octokit.paginate(octokit.repos.listForOrg, {
-            org: org,
-            per_page: 100,
-            request: {
-                signal
-            }
-        });
-
-        const duration = Date.now() - start;
-        logger.debug(`Found ${result.length} in org ${org} in ${duration}ms.`);
-
-        return result;
+        try {
+            logger.debug(`Fetching repository info for org ${org}...`);
+
+            const { durationMs, data } = await measure(() => octokit.paginate(octokit.repos.listForOrg, {
+                org: org,
+                per_page: 100,
+                request: {
+                    signal
+                }
+            }));
+
+            logger.debug(`Found ${data.length} in org ${org} in ${durationMs}ms.`);
+            return data;
+        } catch (e) {
+            logger.error(`Failed to fetch repository info for org ${org}.`, e);
+            return [];
+        }
     }))).flat();
 
     return repos;
 }
 
 const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal) => {
-    const repos = await Promise.all(repoList.map(async (repo) => {
-        logger.debug(`Fetching repository info for ${repo}...`);
-        const start = Date.now();
-
-        const [owner, repoName] = repo.split('/');
-        const result = await octokit.repos.get({
-            owner,
-            repo: repoName,
-            request: {
-                signal
-            }
-        });
-
-        const duration = Date.now() - start;
-        logger.debug(`Found info for repository ${repo} in ${duration}ms`);
-
-        return result.data;
-    }));
+    const repos = (await Promise.all(repoList.map(async (repo) => {
+        try {
+            logger.debug(`Fetching repository info for ${repo}...`);
+
+            const [owner, repoName] = repo.split('/');
+            const { durationMs, data: result } = await measure(() => octokit.repos.get({
+                owner,
+                repo: repoName,
+                request: {
+                    signal
+                }
+            }));
+
+            logger.debug(`Found info for repository ${repo} in ${durationMs}ms`);
+
+            return [result.data];
+        } catch (e) {
+            logger.error(`Failed to fetch repository info for ${repo}.`, e);
+            return [];
+        }
+    }))).flat();
 
     return repos;
 }