Gerrit sync

README.md

@@ -30,7 +30,7 @@ https://github.com/user-attachments/assets/98d46192-5469-430f-ad9e-5c042adbb10d
 
 ## Features
 - 💻 **One-command deployment**: Get started instantly using Docker on your own machine.
-- 🔍 **Multi-repo search**: Effortlessly index and search through multiple public and private repositories in GitHub, GitLab, or Gitea.
+- 🔍 **Multi-repo search**: Effortlessly index and search through multiple public and private repositories in GitHub, GitLab, Gitea, or Gerrit.
 - ⚡**Lightning fast performance**: Built on top of the powerful [Zoekt](https://github.com/sourcegraph/zoekt) search engine.
 - 📂 **Full file visualization**: Instantly view the entire file when selecting any search result.
 - 🎨 **Modern web app**: Enjoy a sleek interface with features like syntax highlighting, light/dark mode, and vim-style navigation 
@@ -62,7 +62,7 @@ Sourcebot supports indexing and searching through public and private repositorie
 <picture>
     <source media="(prefers-color-scheme: dark)" srcset=".github/images/github-favicon-inverted.png">
     <img src="https://github.com/favicon.ico" width="16" height="16" alt="GitHub icon">
-</picture> GitHub, <img src="https://gitlab.com/favicon.ico" width="16" height="16" /> GitLab and <img src="https://gitea.com/favicon.ico" width="16" height="16"> Gitea. This section will guide you through configuring the repositories that Sourcebot indexes. 
+</picture> GitHub, <img src="https://gitlab.com/favicon.ico" width="16" height="16" /> GitLab, <img src="https://gitea.com/favicon.ico" width="16" height="16"> Gitea, and <img src="https://gerrit-review.googlesource.com/favicon.ico" width="16" height="16"> Gerrit. This section will guide you through configuring the repositories that Sourcebot indexes. 
 
 1. Create a new folder on your machine that stores your configs and `.sourcebot` cache, and navigate into it:
     ```sh
@@ -261,6 +261,12 @@ docker run -e <b>GITEA_TOKEN=my-secret-token</b> /* additional args */ ghcr.io/s
 
 </details>
 
+<details>
+<summary><img src="https://gerrit-review.googlesource.com/favicon.ico" width="16" height="16"> Gerrit</summary>
+Gerrit authentication is not yet currently supported.
+</details>
+
+
 </div>
 
 ## Using a self-hosted GitLab / GitHub instance
@@ -397,4 +403,4 @@ NEXT_PUBLIC_SOURCEBOT_TELEMETRY_DISABLED=1
 
 Sourcebot makes use of the following libraries:
 
-- [@vscode/codicons](https://github.com/microsoft/vscode-codicons) under the [CC BY 4.0 License](https://github.com/microsoft/vscode-codicons/blob/main/LICENSE).
+- [@vscode/codicons](https://github.com/microsoft/vscode-codicons) under the [CC BY 4.0 License](https://github.com/microsoft/vscode-codicons/blob/main/LICENSE).

packages/backend/src/gerrit.ts

@@ -0,0 +1,109 @@
+import fetch from 'cross-fetch';
+import { GerritConfig } from './schemas/v2.js';
+import { AppContext, GitRepository } from './types.js';
+import { createLogger } from './logger.js';
+import path from 'path';
+import { measure, marshalBool, excludeReposByName, includeReposByName } from './utils.js';
+
+// https://gerrit-review.googlesource.com/Documentation/rest-api.html
+interface GerritProjects {
+   [projectName: string]: GerritProjectInfo;
+}
+
+interface GerritProjectInfo {
+   id: string;
+   state?: string;
+   web_links?: GerritWebLink[];
+}
+
+interface GerritWebLink {
+   name: string;
+   url: string;
+}
+
+const logger = createLogger('Gerrit');
+
+export const getGerritReposFromConfig = async (config: GerritConfig, ctx: AppContext) => {
+
+   const url = config.url.endsWith('/') ? config.url : `${config.url}/`;
+   const hostname = new URL(config.url).hostname;
+
+   const { durationMs, data: projects } = await measure(() =>
+      fetchAllProjects(url)
+   );
+
+   // exclude "All-Projects" and "All-Users" projects
+   delete projects['All-Projects'];
+   delete projects['All-Users'];
+
+   logger.debug(`Fetched ${Object.keys(projects).length} projects in ${durationMs}ms.`);
+
+   let repos: GitRepository[] = Object.keys(projects).map((projectName) => {
+      const project = projects[projectName];
+      let webUrl = "https://www.gerritcodereview.com/";
+      // Gerrit projects can have multiple web links; use the first one
+      if (project.web_links) {
+         const webLink = project.web_links[0];
+         if (webLink) {
+            webUrl = webLink.url;
+         }
+      }
+      const repoId = `${hostname}/${projectName}`;
+      const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));
+
+      const cloneUrl = `${url}${encodeURIComponent(projectName)}`;
+
+      return {
+         vcs: 'git',
+         codeHost: 'gerrit',
+         name: projectName,
+         id: repoId,
+         cloneUrl: cloneUrl,
+         path: repoPath,
+         isStale: false, // Gerrit projects are typically not stale
+         isFork: false, // Gerrit doesn't have forks in the same way as GitHub
+         isArchived: false,
+         gitConfigMetadata: {
+            // Gerrit uses Gitiles for web UI. This can sometimes be "browse" type in zoekt
+            'zoekt.web-url-type': 'gitiles',
+            'zoekt.web-url': webUrl,
+            'zoekt.name': repoId,
+            'zoekt.archived': marshalBool(false),
+            'zoekt.fork': marshalBool(false),
+            'zoekt.public': marshalBool(true), // Assuming projects are public; adjust as needed
+         },
+         branches: [],
+         tags: []
+      } satisfies GitRepository;
+   });
+
+   // include repos by glob if specified in config
+   if (config.projects) {
+      repos = includeReposByName(repos, config.projects);
+   }
+
+   if (config.exclude && config.exclude.projects) {
+      repos = excludeReposByName(repos, config.exclude.projects);
+   }
+
+   return repos;
+};
+
+const fetchAllProjects = async (url: string): Promise<GerritProjects> => {
+
+   const projectsEndpoint = `${url}projects/`;
+   logger.debug(`Fetching projects from Gerrit at ${projectsEndpoint}...`);
+   const response = await fetch(projectsEndpoint);
+
+   if (!response.ok) {
+      throw new Error(`Failed to fetch projects from Gerrit: ${response.statusText}`);
+   }
+
+   const text = await response.text();
+
+   // Gerrit prepends ")]}'\n" to prevent XSSI attacks; remove it
+   // https://gerrit-review.googlesource.com/Documentation/rest-api.html
+   const jsonText = text.replace(")]}'\n", '');
+   const data = JSON.parse(jsonText);
+   return data;
+};

packages/backend/src/main.ts

@@ -4,6 +4,7 @@ import { SourcebotConfigurationSchema } from "./schemas/v2.js";
 import { getGitHubReposFromConfig } from "./github.js";
 import { getGitLabReposFromConfig } from "./gitlab.js";
 import { getGiteaReposFromConfig } from "./gitea.js";
+import { getGerritReposFromConfig } from "./gerrit.js";
 import { AppContext, LocalRepository, GitRepository, Repository } from "./types.js";
 import { cloneRepository, fetchRepository } from "./git.js";
 import { createLogger } from "./logger.js";
@@ -139,6 +140,11 @@ const syncConfig = async (configPath: string, db: Database, signal: AbortSignal,
                 configRepos.push(...giteaRepos);
                 break;
             }
+            case 'gerrit': {
+                const gerritRepos = await getGerritReposFromConfig(repoConfig, ctx);
+                configRepos.push(...gerritRepos);
+                break;
+            }
             case 'local': {
                 const repo = getLocalRepoFromConfig(repoConfig, ctx);
                 configRepos.push(repo);

packages/backend/src/schemas/v2.ts

@@ -1,6 +1,6 @@
 // THIS IS A AUTO-GENERATED FILE. DO NOT MODIFY MANUALLY!
 
-export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | LocalConfig;
+export type Repos = GitHubConfig | GitLabConfig | GiteaConfig | GerritConfig | LocalConfig;
 
 /**
  * A Sourcebot configuration file outlines which repositories Sourcebot should sync and index.
@@ -173,6 +173,27 @@ export interface GiteaConfig {
   };
   revisions?: GitRevisions;
 }
+export interface GerritConfig {
+  /**
+   * Gerrit Configuration
+   */
+  type: "gerrit";
+  /**
+   * The URL of the Gerrit host.
+   */
+  url: string;
+  /**
+   * List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported
+   */
+  projects?: string[];
+  exclude?: {
+    /**
+     * List of specific projects to exclude from syncing.
+     */
+    projects?: string[];
+  };
+  revisions?: GitRevisions;
+}
 export interface LocalConfig {
   /**
    * Local Configuration

packages/backend/src/utils.ts

@@ -48,6 +48,16 @@ export const excludeReposByName = <T extends Repository>(repos: T[], excludedRep
     });
 }
 
+export const includeReposByName = <T extends Repository>(repos: T[], includedRepoNames: string[], logger?: Logger) => {
+    return repos.filter((repo) => {
+        if (micromatch.isMatch(repo.name, includedRepoNames)) {
+            logger?.debug(`Including repo ${repo.id}. Reason: repos contain ${repo.name}`);
+            return true;
+        }
+        return false;
+    });
+}
+
 export const getTokenFromConfig = (token: string | { env: string }, ctx: AppContext) => {
     if (typeof token === 'string') {
         return token;

packages/web/src/lib/utils.ts

@@ -3,6 +3,7 @@ import { twMerge } from "tailwind-merge"
 import githubLogo from "../../public/github.svg";
 import gitlabLogo from "../../public/gitlab.svg";
 import giteaLogo from "../../public/gitea.svg";
+import gerritLogo from "../../public/gerrit.svg";
 import { ServiceError } from "./serviceError";
 import { Repository } from "./types";
 
@@ -31,7 +32,7 @@ export const createPathWithQueryParams = (path: string, ...queryParams: [string,
 }
 
 type CodeHostInfo = {
-    type: "github" | "gitlab" | "gitea";
+    type: "github" | "gitlab" | "gitea" | "gerrit";
     displayName: string;
     costHostName: string;
     repoLink: string;
@@ -44,15 +45,14 @@ export const getRepoCodeHostInfo = (repo?: Repository): CodeHostInfo | undefined
         return undefined;
     }
 
-    const hostType = repo.RawConfig ? repo.RawConfig['web-url-type'] : undefined;
-    if (!hostType) {
+    const webUrlType = repo.RawConfig ? repo.RawConfig['web-url-type'] : undefined;
+    if (!webUrlType) {
         return undefined;
     }
 
     const url = new URL(repo.URL);
     const displayName = url.pathname.slice(1);
-
-    switch (hostType) {
+    switch (webUrlType) {
         case 'github':
             return {
                 type: "github",
@@ -78,6 +78,14 @@ export const getRepoCodeHostInfo = (repo?: Repository): CodeHostInfo | undefined
                 repoLink: repo.URL,
                 icon: giteaLogo,
             }
+        case 'gitiles':
+            return {
+                type: "gerrit",
+                displayName: displayName,
+                costHostName: "Gerrit",
+                repoLink: repo.URL,
+                icon: gerritLogo,
+            }
     }
 }
 
@@ -113,4 +121,4 @@ export const base64Decode = (base64: string): string => {
 // @see: https://stackoverflow.com/a/65959350/23221295
 export const isDefined = <T>(arg: T | null | undefined): arg is T extends null | undefined ? never : T => {
     return arg !== null && arg !== undefined;
-}
+}

schemas/v2/index.json

@@ -356,6 +356,61 @@
             ],
             "additionalProperties": false
         },
+        "GerritConfig": {
+            "type": "object",
+            "properties": {
+                "type": {
+                    "const": "gerrit",
+                    "description": "Gerrit Configuration"
+                },
+                "url": {
+                    "type": "string",
+                    "format": "url",
+                    "description": "The URL of the Gerrit host.",
+                    "examples": [
+                        "https://gerrit.example.com"
+                    ],
+                    "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
+                },
+                "projects": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    },
+                    "description": "List of specific projects to sync. If not specified, all projects will be synced. Glob patterns are supported",
+                    "examples": [
+                        [
+                            "project1/repo1",
+                            "project2/**"
+                        ]
+                    ]
+                },
+                "exclude": {
+                    "type": "object",
+                    "properties": {
+                        "projects": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            },
+                            "examples": [
+                                [
+                                    "project1/repo1",
+                                    "project2/**"
+                                ]
+                            ],
+                            "description": "List of specific projects to exclude from syncing."
+                        }
+                    },
+                    "additionalProperties": false
+                }
+            },
+            "required": [
+                "type",
+                "url"
+            ],
+            "additionalProperties": false
+        },
         "LocalConfig": {
             "type": "object",
             "properties": {
@@ -415,6 +470,9 @@
                 {
                     "$ref": "#/definitions/GiteaConfig"
                 },
+                {
+                    "$ref": "#/definitions/GerritConfig"
+                },
                 {
                     "$ref": "#/definitions/LocalConfig"
                 }