Add config param all to enable syncing all projects in GitLab instance

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added `DOMAIN_SUB_PATH` environment variable to allow overriding the default domain subpath. ([#74](https://github.com/sourcebot-dev/sourcebot/pull/74))
+- Added option `all` to the GitLab index schema, allowing for indexing all projects in a self-hosted GitLab instance. ([#84](https://github.com/sourcebot-dev/sourcebot/pull/84)) 
 
 ## [2.4.3] - 2024-11-18
 

configs/basic.json

@@ -59,6 +59,15 @@
         {
             "type": "local",
             "path": "/path/to/local/repo"
+        },
+        // Index all projects in a self-hosted GitLab instance
+        // that are visible to the provided token.
+        // Note: this does not work with gitlab.com
+        {
+            "type": "gitlab",
+            "url": "https://gitlab.example.com",
+            "token": "my-token",
+            "all": true
         }
     ]
 }

packages/backend/src/gitlab.ts

@@ -7,6 +7,7 @@ import path from 'path';
 import micromatch from "micromatch";
 
 const logger = createLogger("GitLab");
+const GITLAB_CLOUD_HOSTNAME = "gitlab.com";
 
 export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppContext) => {
     const token = config.token ? getTokenFromConfig(config.token, ctx) : undefined;
@@ -18,9 +19,24 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
             host: config.url,
         } : {}),
     });
+    const hostname = config.url ? new URL(config.url).hostname : GITLAB_CLOUD_HOSTNAME;
+
 
     let allProjects: ProjectSchema[] = [];
 
+    if (config.all === true) {
+        if (hostname !== GITLAB_CLOUD_HOSTNAME) {
+            logger.debug(`Fetching all projects visible in ${config.url}...`);
+            const { durationMs, data: _projects } = await measure(() => api.Projects.all({
+                perPage: 100,
+            }));
+            logger.debug(`Found ${_projects.length} projects in ${durationMs}ms.`);
+            allProjects = allProjects.concat(_projects);
+        } else {
+            logger.warn(`Ignoring option all:true in ${ctx.configPath} : host is ${GITLAB_CLOUD_HOSTNAME}`);
+        }
+    }
+
     if (config.groups) {
         const _projects = (await Promise.all(config.groups.map(async (group) => {
             logger.debug(`Fetching project info for group ${group}...`);
@@ -62,7 +78,6 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
 
     let repos: GitRepository[] = allProjects
         .map((project) => {
-            const hostname = config.url ? new URL(config.url).hostname : "gitlab.com";
             const repoId = `${hostname}/${project.path_with_namespace}`;
             const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`))
             const isFork = project.forked_from_project !== undefined;

packages/backend/src/schemas/v2.ts

@@ -93,6 +93,10 @@ export interface GitLabConfig {
    * The URL of the GitLab host. Defaults to https://gitlab.com
    */
   url?: string;
+  /**
+   * Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com .
+   */
+  all?: boolean;
   /**
    * List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property.
    */

schemas/v2/index.json

@@ -189,6 +189,11 @@
                     ],
                     "pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
                 },
+                "all": {
+                    "type": "boolean",
+                    "default": false,
+                    "description": "Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com ."
+                },
                 "users": {
                     "type": "array",
                     "items": {