chore(worker,web): Repo indexing stability improvements + perf improvements to web

.env.development

@@ -80,9 +80,6 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
 # Controls the number of concurrent indexing jobs that can run at once
 # INDEX_CONCURRENCY_MULTIPLE=
 
-# Controls the polling interval for the web app
-# NEXT_PUBLIC_POLLING_INTERVAL_MS=
-
 # Controls the version of the web app
 # NEXT_PUBLIC_SOURCEBOT_VERSION=
 

CHANGELOG.md

@@ -14,10 +14,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - Fixed "dubious ownership" errors when cloning / fetching repos. [#553](https://github.com/sourcebot-dev/sourcebot/pull/553)
+- Fixed issue with Ask Sourcebot tutorial re-appearing after restarting the browser. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
 
 ### Changed
-- Remove spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
 - Improved search performance for unbounded search queries. [#555](https://github.com/sourcebot-dev/sourcebot/pull/555)
+- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
+- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
+- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
+
+### Removed
+- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
+- Removed connections management page. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
 
 ### Added
 - Added support for passing db connection url as seperate `DATABASE_HOST`, `DATABASE_USERNAME`, `DATABASE_PASSWORD`, `DATABASE_NAME`, and `DATABASE_ARGS` env vars. [#545](https://github.com/sourcebot-dev/sourcebot/pull/545)

package.json

@@ -6,7 +6,7 @@
     "scripts": {
         "build": "cross-env SKIP_ENV_VALIDATION=1 yarn workspaces foreach -A run build",
         "test": "yarn workspaces foreach -A run test",
-        "dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web watch:mcp watch:schemas",
+        "dev": "concurrently --kill-others --names \"zoekt,worker,web,mcp,schemas\" 'yarn dev:zoekt' 'yarn dev:backend' 'yarn dev:web' 'yarn watch:mcp' 'yarn watch:schemas'",
         "with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
         "dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
         "dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
@@ -21,9 +21,9 @@
         "build:deps": "yarn workspaces foreach -R --from '{@sourcebot/schemas,@sourcebot/error,@sourcebot/crypto,@sourcebot/db,@sourcebot/shared}' run build"
     },
     "devDependencies": {
+        "concurrently": "^9.2.1",
         "cross-env": "^7.0.3",
-        "dotenv-cli": "^8.0.0",
-        "npm-run-all": "^4.1.5"
+        "dotenv-cli": "^8.0.0"
     },
     "packageManager": "yarn@4.7.0",
     "resolutions": {

packages/backend/package.json

@@ -45,6 +45,7 @@
         "git-url-parse": "^16.1.0",
         "gitea-js": "^1.22.0",
         "glob": "^11.0.0",
+        "groupmq": "^1.0.0",
         "ioredis": "^5.4.2",
         "lowdb": "^7.0.1",
         "micromatch": "^4.0.8",

packages/backend/src/connectionManager.ts

@@ -364,12 +364,12 @@ export class ConnectionManager {
         }
     }
 
-    public dispose() {
+    public async dispose() {
         if (this.interval) {
             clearInterval(this.interval);
         }
-        this.worker.close();
-        this.queue.close();
+        await this.worker.close();
+        await this.queue.close();
     }
 }
 

packages/backend/src/constants.ts

@@ -1,4 +1,6 @@
+import { env } from "./env.js";
 import { Settings } from "./types.js";
+import path from "path";
 
 /**
  * Default settings.
@@ -22,4 +24,7 @@ export const DEFAULT_SETTINGS: Settings = {
 
 export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
     'github',
-];
+];
+
+export const REPOS_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'repos');
+export const INDEX_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'index');

packages/backend/src/ee/repoPermissionSyncer.ts

@@ -101,12 +101,12 @@ export class RepoPermissionSyncer {
         }, 1000 * 5);
     }
 
-    public dispose() {
+    public async dispose() {
         if (this.interval) {
             clearInterval(this.interval);
         }
-        this.worker.close();
-        this.queue.close();
+        await this.worker.close();
+        await this.queue.close();
     }
 
     private async schedulePermissionSync(repos: Repo[]) {

packages/backend/src/ee/userPermissionSyncer.ts

@@ -101,12 +101,12 @@ export class UserPermissionSyncer {
         }, 1000 * 5);
     }
 
-    public dispose() {
+    public async dispose() {
         if (this.interval) {
             clearInterval(this.interval);
         }
-        this.worker.close();
-        this.queue.close();
+        await this.worker.close();
+        await this.queue.close();
     }
 
     private async schedulePermissionSync(users: User[]) {

packages/backend/src/env.ts

@@ -44,6 +44,7 @@ export const env = createEnv({
         LOGTAIL_TOKEN: z.string().optional(),
         LOGTAIL_HOST: z.string().url().optional(),
         SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"),
+        DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'),
 
         DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
         CONFIG_PATH: z.string().optional(),

packages/backend/src/git.ts

@@ -1,30 +1,67 @@
 import { CheckRepoActions, GitConfigScope, simpleGit, SimpleGitProgressEvent } from 'simple-git';
 import { mkdir } from 'node:fs/promises';
 import { env } from './env.js';
+import { dirname, resolve } from 'node:path';
+import { existsSync } from 'node:fs';
 
 type onProgressFn = (event: SimpleGitProgressEvent) => void;
 
+/**
+ * Creates a simple-git client that has it's working directory
+ * set to the given path.
+ */
+const createGitClientForPath = (path: string, onProgress?: onProgressFn, signal?: AbortSignal) => {
+    if (!existsSync(path)) {
+        throw new Error(`Path ${path} does not exist`);
+    }
+
+    const parentPath = resolve(dirname(path));
+
+    const git = simpleGit({
+        progress: onProgress,
+        abort: signal,
+    })
+        .env({
+            ...process.env,
+            /**
+             * @note on some inside-baseball on why this is necessary: The specific
+             * issue we saw was that a `git clone` would fail without throwing, and
+             * then a subsequent `git config` command would run, but since the clone
+             * failed, it wouldn't be running in a git directory. Git would then walk
+             * up the directory tree until it either found a git directory (in the case
+             * of the development env) or it would hit a GIT_DISCOVERY_ACROSS_FILESYSTEM
+             * error when trying to cross a filesystem boundary (in the prod case).
+             * GIT_CEILING_DIRECTORIES ensures that this walk will be limited to the
+             * parent directory.
+             */
+            GIT_CEILING_DIRECTORIES: parentPath,
+        })
+        .cwd({
+            path,
+        });
+
+    return git;
+}
+
 export const cloneRepository = async (
     {
         cloneUrl,
         authHeader,
         path,
         onProgress,
+        signal,
     }: {
         cloneUrl: string,
         authHeader?: string,
         path: string,
         onProgress?: onProgressFn
+        signal?: AbortSignal
     }
 ) => {
     try {
         await mkdir(path, { recursive: true });
 
-        const git = simpleGit({
-            progress: onProgress,
-        }).cwd({
-            path,
-        })
+        const git = createGitClientForPath(path, onProgress, signal);
 
         const cloneArgs = [
             "--bare",
@@ -33,7 +70,11 @@ export const cloneRepository = async (
 
         await git.clone(cloneUrl, path, cloneArgs);
 
-        await unsetGitConfig(path, ["remote.origin.url"]);
+        await unsetGitConfig({
+            path,
+            keys: ["remote.origin.url"],
+            signal,
+        });
     } catch (error: unknown) {
         const baseLog = `Failed to clone repository: ${path}`;
 
@@ -54,20 +95,17 @@ export const fetchRepository = async (
         authHeader,
         path,
         onProgress,
+        signal,
     }: {
         cloneUrl: string,
         authHeader?: string,
         path: string,
-        onProgress?: onProgressFn
+        onProgress?: onProgressFn,
+        signal?: AbortSignal
     }
 ) => {
+    const git = createGitClientForPath(path, onProgress, signal);
     try {
-        const git = simpleGit({
-            progress: onProgress,
-        }).cwd({
-            path: path,
-        })
-
         if (authHeader) {
             await git.addConfig("http.extraHeader", authHeader);
         }
@@ -90,12 +128,6 @@ export const fetchRepository = async (
         }
     } finally {
         if (authHeader) {
-            const git = simpleGit({
-                progress: onProgress,
-            }).cwd({
-                path: path,
-            })
-
             await git.raw(["config", "--unset", "http.extraHeader", authHeader]);
         }
     }
@@ -107,10 +139,19 @@ export const fetchRepository = async (
  * that do not exist yet. It will _not_ remove any existing keys that are not
  * present in gitConfig.
  */
-export const upsertGitConfig = async (path: string, gitConfig: Record<string, string>, onProgress?: onProgressFn) => {
-    const git = simpleGit({
-        progress: onProgress,
-    }).cwd(path);
+export const upsertGitConfig = async (
+    {
+        path,
+        gitConfig,
+        onProgress,
+        signal,
+    }: {
+        path: string,
+        gitConfig: Record<string, string>,
+        onProgress?: onProgressFn,
+        signal?: AbortSignal
+    }) => {
+    const git = createGitClientForPath(path, onProgress, signal);
 
     try {
         for (const [key, value] of Object.entries(gitConfig)) {
@@ -129,10 +170,19 @@ export const upsertGitConfig = async (path: string, gitConfig: Record<string, st
  * Unsets the specified keys in the git config for the repo at the given path.
  * If a key is not set, this is a no-op.
  */
-export const unsetGitConfig = async (path: string, keys: string[], onProgress?: onProgressFn) => {
-    const git = simpleGit({
-        progress: onProgress,
-    }).cwd(path);
+export const unsetGitConfig = async (
+    {
+        path,
+        keys,
+        onProgress,
+        signal,
+    }: {
+        path: string,
+        keys: string[],
+        onProgress?: onProgressFn,
+        signal?: AbortSignal
+    }) => {
+    const git = createGitClientForPath(path, onProgress, signal);
 
     try {
         const configList = await git.listConfig();
@@ -155,10 +205,20 @@ export const unsetGitConfig = async (path: string, keys: string[], onProgress?:
 /**
  * Returns true if `path` is the _root_ of a git repository.
  */
-export const isPathAValidGitRepoRoot = async (path: string, onProgress?: onProgressFn) => {
-    const git = simpleGit({
-        progress: onProgress,
-    }).cwd(path);
+export const isPathAValidGitRepoRoot = async ({
+    path,
+    onProgress,
+    signal,
+}: {
+    path: string,
+    onProgress?: onProgressFn,
+    signal?: AbortSignal
+}) => {
+    if (!existsSync(path)) {
+        return false;
+    }
+
+    const git = createGitClientForPath(path, onProgress, signal);
 
     try {
         return git.checkIsRepo(CheckRepoActions.IS_REPO_ROOT);
@@ -184,7 +244,7 @@ export const isUrlAValidGitRepo = async (url: string) => {
 }
 
 export const getOriginUrl = async (path: string) => {
-    const git = simpleGit().cwd(path);
+    const git = createGitClientForPath(path);
 
     try {
         const remotes = await git.getConfig('remote.origin.url', GitConfigScope.local);
@@ -199,18 +259,13 @@ export const getOriginUrl = async (path: string) => {
 }
 
 export const getBranches = async (path: string) => {
-    const git = simpleGit();
-    const branches = await git.cwd({
-        path,
-    }).branch();
-
+    const git = createGitClientForPath(path);
+    const branches = await git.branch();
     return branches.all;
 }
 
 export const getTags = async (path: string) => {
-    const git = simpleGit();
-    const tags = await git.cwd({
-        path,
-    }).tags();
+    const git = createGitClientForPath(path);
+    const tags = await git.tags();
     return tags.all;
 }