From 32ce1ca16dc99a9df045c6397a9e63ed352f8507 Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 9 Oct 2025 20:10:29 -0700 Subject: [PATCH 01/24] wip --- packages/backend/package.json | 3 + packages/backend/src/index.ts | 6 +- packages/backend/src/indexSyncer.ts | 225 ++++++++++++++++++++++++++++ packages/db/prisma/schema.prisma | 24 ++- yarn.lock | 171 ++++++++++++++++----- 5 files changed, 390 insertions(+), 39 deletions(-) create mode 100644 packages/backend/src/indexSyncer.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index dade7893..4bd3fe40 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -22,6 +22,8 @@ "vitest": "^2.1.9" }, "dependencies": { + "@bull-board/api": "^6.13.0", + "@bull-board/express": "^6.13.0", "@coderabbitai/bitbucket": "^1.1.3", "@gitbeaker/rest": "^40.5.1", "@octokit/rest": "^21.0.2", @@ -45,6 +47,7 @@ "git-url-parse": "^16.1.0", "gitea-js": "^1.22.0", "glob": "^11.0.0", + "groupmq": "^1.0.0", "ioredis": "^5.4.2", "lowdb": "^7.0.1", "micromatch": "^4.0.8", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 93f95e0b..3850d07b 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -15,6 +15,7 @@ import { PromClient } from './promClient.js'; import { RepoManager } from './repoManager.js'; import { AppContext } from "./types.js"; import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js"; +import { IndexSyncer } from "./indexSyncer.js"; const logger = createLogger('backend-entrypoint'); @@ -71,11 +72,13 @@ const connectionManager = new ConnectionManager(prisma, settings, redis); const repoManager = new RepoManager(prisma, settings, redis, promClient, context); const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); const userPermissionSyncer = new UserPermissionSyncer(prisma, settings, redis); +const indexSyncer = new IndexSyncer(prisma, settings, redis); await repoManager.validateIndexedReposHaveShards(); connectionManager.startScheduler(); -repoManager.startScheduler(); +// repoManager.startScheduler(); +indexSyncer.startScheduler(); if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && !hasEntitlement('permission-syncing')) { logger.error('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.'); @@ -93,6 +96,7 @@ const cleanup = async (signal: string) => { repoManager.dispose(); repoPermissionSyncer.dispose(); userPermissionSyncer.dispose(); + indexSyncer.dispose(); await prisma.$disconnect(); await redis.quit(); diff --git a/packages/backend/src/indexSyncer.ts b/packages/backend/src/indexSyncer.ts new file mode 100644 index 00000000..11a88009 --- /dev/null +++ b/packages/backend/src/indexSyncer.ts @@ -0,0 +1,225 @@ +import { createBullBoard } from '@bull-board/api'; +import { ExpressAdapter } from '@bull-board/express'; +import * as Sentry from '@sentry/node'; +import { PrismaClient, Repo, RepoIndexingJobStatus } from "@sourcebot/db"; +import { createLogger } from "@sourcebot/logger"; +import express from 'express'; +import { BullBoardGroupMQAdapter, Queue, Worker } from "groupmq"; +import { Redis } from 'ioredis'; +import { Settings } from "./types.js"; + +const logger = createLogger('index-syncer'); + +type IndexSyncJob = { + jobId: string; +} + +const JOB_TIMEOUT_MS = 1000 * 60; // 60 second timeout. + +export class IndexSyncer { + private interval?: NodeJS.Timeout; + private queue: Queue; + private worker: Worker; + + constructor( + private db: PrismaClient, + private settings: Settings, + redis: Redis, + ) { + this.queue = new Queue({ + redis, + namespace: 'index-sync-queue', + jobTimeoutMs: JOB_TIMEOUT_MS, + // logger: true, + }); + + this.worker = new Worker({ + queue: this.queue, + maxStalledCount: 1, + stalledInterval: 1000, + handler: async (job) => { + const id = job.data.jobId; + const { repo } = await this.db.repoIndexingJob.update({ + where: { + id, + }, + data: { + status: RepoIndexingJobStatus.IN_PROGRESS, + }, + select: { + repo: true, + } + }); + + logger.info(`Running index job ${id} for repo ${repo.name}`); + + await new Promise(resolve => setTimeout(resolve, 1000 * 10)); + + return true; + }, + concurrency: 4, + }); + + this.worker.on('completed', async (job) => { + const { repo } = await this.db.repoIndexingJob.update({ + where: { + id: job.data.jobId, + }, + data: { + status: RepoIndexingJobStatus.COMPLETED, + repo: { + update: { + indexedAt: new Date(), + } + }, + completedAt: new Date(), + }, + select: { + repo: true, + } + }); + + logger.info(`Completed index job ${job.data.jobId} for repo ${repo.name}`); + }); + + this.worker.on('failed', async (job) => { + const { repo } = await this.db.repoIndexingJob.update({ + where: { + id: job.data.jobId, + }, + data: { + status: RepoIndexingJobStatus.FAILED, + completedAt: new Date(), + errorMessage: job.failedReason, + }, + select: { + repo: true, + } + }); + + logger.error(`Failed index job ${job.data.jobId} for repo ${repo.name}`); + }); + + this.worker.on('stalled', async (jobId) => { + const { repo } = await this.db.repoIndexingJob.update({ + where: { id: jobId }, + data: { + status: RepoIndexingJobStatus.FAILED, + completedAt: new Date(), + errorMessage: 'Job stalled', + }, + select: { repo: true } + }); + + logger.warn(`Job ${jobId} stalled for repo ${repo.name}`); + }); + + this.worker.on('error', async (error) => { + Sentry.captureException(error); + logger.error(`Index syncer worker error.`, error); + }); + + const app = express(); + const serverAdapter = new ExpressAdapter(); + + createBullBoard({ + queues: [new BullBoardGroupMQAdapter(this.queue, { displayName: 'Index Sync' })], + serverAdapter, + }); + + app.use('/', serverAdapter.getRouter()); + app.listen(3070); + } + + public async startScheduler() { + this.interval = setInterval(async () => { + const thresholdDate = new Date(Date.now() - this.settings.reindexIntervalMs); + + const repos = await this.db.repo.findMany({ + where: { + AND: [ + { + OR: [ + { indexedAt: null }, + { indexedAt: { lt: thresholdDate } }, + ] + }, + { + NOT: { + indexingJobs: { + some: { + OR: [ + // Don't schedule if there are active jobs that were created within the threshold date. + // This handles the case where a job is stuck in a pending state and will never be scheduled. + { + AND: [ + { + status: { + in: [ + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, + ] + }, + }, + { + createdAt: { + gt: thresholdDate, + } + } + ] + }, + // Don't schedule if there are recent failed jobs (within the threshold date). + { + AND: [ + { status: RepoIndexingJobStatus.FAILED }, + { completedAt: { gt: thresholdDate } }, + ] + } + ] + } + } + } + } + ], + } + }); + + if (repos.length === 0) { + return; + } + + await this.scheduleIndexSync(repos); + }, 1000 * 5); + + this.worker.run(); + } + + private async scheduleIndexSync(repos: Repo[]) { + // @note: we don't perform this in a transaction because + // we want to avoid the situation where a job is created and run + // prior to the transaction being committed. + const jobs = await this.db.repoIndexingJob.createManyAndReturn({ + data: repos.map(repo => ({ + repoId: repo.id, + })) + }); + + for (const job of jobs) { + await this.queue.add({ + groupId: `repo:${job.repoId}`, + data: { + jobId: job.id, + }, + jobId: job.id, + }); + } + } + + public dispose() { + if (this.interval) { + clearInterval(this.interval); + } + this.worker.close(); + this.queue.close(); + } +} \ No newline at end of file diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index bdebbc69..75e30ec9 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -46,7 +46,6 @@ model Repo { displayName String? /// Display name of the repo for UI (ex. sourcebot-dev/sourcebot) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - indexedAt DateTime? /// When the repo was last indexed successfully. isFork Boolean isArchived Boolean isPublic Boolean @default(false) @@ -61,6 +60,9 @@ model Repo { permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. + indexingJobs RepoIndexingJob[] + indexedAt DateTime? /// When the repo was last indexed successfully. + external_id String /// The id of the repo in the external service external_codeHostType String /// The type of the external service (e.g., github, gitlab, etc.) external_codeHostUrl String /// The base url of the external service (e.g., https://github.com) @@ -74,6 +76,26 @@ model Repo { @@index([orgId]) } +enum RepoIndexingJobStatus { + PENDING + IN_PROGRESS + COMPLETED + FAILED +} + +model RepoIndexingJob { + id String @id @default(cuid()) + status RepoIndexingJobStatus @default(PENDING) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + completedAt DateTime? + + errorMessage String? + + repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) + repoId Int +} + enum RepoPermissionSyncJobStatus { PENDING IN_PROGRESS diff --git a/yarn.lock b/yarn.lock index 7fae191b..6fe84282 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1075,6 +1075,38 @@ __metadata: languageName: node linkType: hard +"@bull-board/api@npm:6.13.0, @bull-board/api@npm:^6.13.0": + version: 6.13.0 + resolution: "@bull-board/api@npm:6.13.0" + dependencies: + redis-info: "npm:^3.1.0" + peerDependencies: + "@bull-board/ui": 6.13.0 + checksum: 10c0/52d02ec30f7858bcbac0dee16d66d0605323e06706af64281adae78b28602aba798f2119be3c7e90d544f3e31dae9c01977fa26655340bb4dae9473b70587d46 + languageName: node + linkType: hard + +"@bull-board/express@npm:^6.13.0": + version: 6.13.0 + resolution: "@bull-board/express@npm:6.13.0" + dependencies: + "@bull-board/api": "npm:6.13.0" + "@bull-board/ui": "npm:6.13.0" + ejs: "npm:^3.1.10" + express: "npm:^4.21.1 || ^5.0.0" + checksum: 10c0/766125212b63d9ac802af863e4710875c876c84c03e29efa45b4fc7835bf4122db89b9b6e7cbc89db0087d5afe61fd476d1701933f75b9ab92537e08a1bb75e2 + languageName: node + linkType: hard + +"@bull-board/ui@npm:6.13.0": + version: 6.13.0 + resolution: "@bull-board/ui@npm:6.13.0" + dependencies: + "@bull-board/api": "npm:6.13.0" + checksum: 10c0/aa755a14113d19d960eaa61d568a56c843e43cb741b3d65868c43d1444783bafdb2b1ebfd9c2f7031bd792f797696fbaabb8b40ab7766e9e0d86b79b1e1a2202 + languageName: node + linkType: hard + "@codemirror/autocomplete@npm:^6.0.0, @codemirror/autocomplete@npm:^6.16.2, @codemirror/autocomplete@npm:^6.3.2, @codemirror/autocomplete@npm:^6.7.1": version: 6.18.6 resolution: "@codemirror/autocomplete@npm:6.18.6" @@ -7544,6 +7576,8 @@ __metadata: version: 0.0.0-use.local resolution: "@sourcebot/backend@workspace:packages/backend" dependencies: + "@bull-board/api": "npm:^6.13.0" + "@bull-board/express": "npm:^6.13.0" "@coderabbitai/bitbucket": "npm:^1.1.3" "@gitbeaker/rest": "npm:^40.5.1" "@octokit/rest": "npm:^21.0.2" @@ -7571,6 +7605,7 @@ __metadata: git-url-parse: "npm:^16.1.0" gitea-js: "npm:^1.22.0" glob: "npm:^11.0.0" + groupmq: "npm:^1.0.0" ioredis: "npm:^5.4.2" json-schema-to-typescript: "npm:^15.0.4" lowdb: "npm:^7.0.1" @@ -9497,7 +9532,7 @@ __metadata: languageName: node linkType: hard -"async@npm:^3.2.3": +"async@npm:^3.2.3, async@npm:^3.2.6": version: 3.2.6 resolution: "async@npm:3.2.6" checksum: 10c0/36484bb15ceddf07078688d95e27076379cc2f87b10c03b6dd8a83e89475a3c8df5848859dd06a4c95af1e4c16fc973de0171a77f18ea00be899aca2a4f85e70 @@ -11144,6 +11179,17 @@ __metadata: languageName: node linkType: hard +"ejs@npm:^3.1.10": + version: 3.1.10 + resolution: "ejs@npm:3.1.10" + dependencies: + jake: "npm:^10.8.5" + bin: + ejs: bin/cli.js + checksum: 10c0/52eade9e68416ed04f7f92c492183340582a36482836b11eab97b159fcdcfdedc62233a1bf0bf5e5e1851c501f2dca0e2e9afd111db2599e4e7f53ee29429ae1 + languageName: node + linkType: hard + "electron-to-chromium@npm:^1.5.73": version: 1.5.123 resolution: "electron-to-chromium@npm:1.5.123" @@ -12199,6 +12245,41 @@ __metadata: languageName: node linkType: hard +"express@npm:^4.21.1 || ^5.0.0, express@npm:^5.0.1, express@npm:^5.1.0": + version: 5.1.0 + resolution: "express@npm:5.1.0" + dependencies: + accepts: "npm:^2.0.0" + body-parser: "npm:^2.2.0" + content-disposition: "npm:^1.0.0" + content-type: "npm:^1.0.5" + cookie: "npm:^0.7.1" + cookie-signature: "npm:^1.2.1" + debug: "npm:^4.4.0" + encodeurl: "npm:^2.0.0" + escape-html: "npm:^1.0.3" + etag: "npm:^1.8.1" + finalhandler: "npm:^2.1.0" + fresh: "npm:^2.0.0" + http-errors: "npm:^2.0.0" + merge-descriptors: "npm:^2.0.0" + mime-types: "npm:^3.0.0" + on-finished: "npm:^2.4.1" + once: "npm:^1.4.0" + parseurl: "npm:^1.3.3" + proxy-addr: "npm:^2.0.7" + qs: "npm:^6.14.0" + range-parser: "npm:^1.2.1" + router: "npm:^2.2.0" + send: "npm:^1.1.0" + serve-static: "npm:^2.2.0" + statuses: "npm:^2.0.1" + type-is: "npm:^2.0.1" + vary: "npm:^1.1.2" + checksum: 10c0/80ce7c53c5f56887d759b94c3f2283e2e51066c98d4b72a4cc1338e832b77f1e54f30d0239cc10815a0f849bdb753e6a284d2fa48d4ab56faf9c501f55d751d6 + languageName: node + linkType: hard + "express@npm:^4.21.2": version: 4.21.2 resolution: "express@npm:4.21.2" @@ -12238,41 +12319,6 @@ __metadata: languageName: node linkType: hard -"express@npm:^5.0.1, express@npm:^5.1.0": - version: 5.1.0 - resolution: "express@npm:5.1.0" - dependencies: - accepts: "npm:^2.0.0" - body-parser: "npm:^2.2.0" - content-disposition: "npm:^1.0.0" - content-type: "npm:^1.0.5" - cookie: "npm:^0.7.1" - cookie-signature: "npm:^1.2.1" - debug: "npm:^4.4.0" - encodeurl: "npm:^2.0.0" - escape-html: "npm:^1.0.3" - etag: "npm:^1.8.1" - finalhandler: "npm:^2.1.0" - fresh: "npm:^2.0.0" - http-errors: "npm:^2.0.0" - merge-descriptors: "npm:^2.0.0" - mime-types: "npm:^3.0.0" - on-finished: "npm:^2.4.1" - once: "npm:^1.4.0" - parseurl: "npm:^1.3.3" - proxy-addr: "npm:^2.0.7" - qs: "npm:^6.14.0" - range-parser: "npm:^1.2.1" - router: "npm:^2.2.0" - send: "npm:^1.1.0" - serve-static: "npm:^2.2.0" - statuses: "npm:^2.0.1" - type-is: "npm:^2.0.1" - vary: "npm:^1.1.2" - checksum: 10c0/80ce7c53c5f56887d759b94c3f2283e2e51066c98d4b72a4cc1338e832b77f1e54f30d0239cc10815a0f849bdb753e6a284d2fa48d4ab56faf9c501f55d751d6 - languageName: node - linkType: hard - "extend@npm:^3.0.0, extend@npm:^3.0.2": version: 3.0.2 resolution: "extend@npm:3.0.2" @@ -12427,6 +12473,15 @@ __metadata: languageName: node linkType: hard +"filelist@npm:^1.0.4": + version: 1.0.4 + resolution: "filelist@npm:1.0.4" + dependencies: + minimatch: "npm:^5.0.1" + checksum: 10c0/426b1de3944a3d153b053f1c0ebfd02dccd0308a4f9e832ad220707a6d1f1b3c9784d6cadf6b2f68f09a57565f63ebc7bcdc913ccf8012d834f472c46e596f41 + languageName: node + linkType: hard + "fill-range@npm:^7.1.1": version: 7.1.1 resolution: "fill-range@npm:7.1.1" @@ -13024,6 +13079,17 @@ __metadata: languageName: node linkType: hard +"groupmq@npm:^1.0.0": + version: 1.0.0 + resolution: "groupmq@npm:1.0.0" + dependencies: + cron-parser: "npm:^4.9.0" + peerDependencies: + ioredis: ">=5" + checksum: 10c0/1795e483fc712ff91a2f7abb32f29f14f94b9d5e32033df064021f284a12773822e241f0d5e061978632dcf20df5db3f3d7701f04e53979d5b594f618fc53a27 + languageName: node + linkType: hard + "gtoken@npm:^7.0.0": version: 7.1.0 resolution: "gtoken@npm:7.1.0" @@ -13995,6 +14061,19 @@ __metadata: languageName: node linkType: hard +"jake@npm:^10.8.5": + version: 10.9.4 + resolution: "jake@npm:10.9.4" + dependencies: + async: "npm:^3.2.6" + filelist: "npm:^1.0.4" + picocolors: "npm:^1.1.1" + bin: + jake: bin/cli.js + checksum: 10c0/bb52f000340d4a32f1a3893b9abe56ef2b77c25da4dbf2c0c874a8159d082dddda50a5ad10e26060198bd645b928ba8dba3b362710f46a247e335321188c5a9c + languageName: node + linkType: hard + "jiti@npm:^1.21.6": version: 1.21.7 resolution: "jiti@npm:1.21.7" @@ -14425,7 +14504,7 @@ __metadata: languageName: node linkType: hard -"lodash@npm:^4.17.21": +"lodash@npm:^4.17.11, lodash@npm:^4.17.21": version: 4.17.21 resolution: "lodash@npm:4.17.21" checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c @@ -15317,6 +15396,15 @@ __metadata: languageName: node linkType: hard +"minimatch@npm:^5.0.1": + version: 5.1.6 + resolution: "minimatch@npm:5.1.6" + dependencies: + brace-expansion: "npm:^2.0.1" + checksum: 10c0/3defdfd230914f22a8da203747c42ee3c405c39d4d37ffda284dac5e45b7e1f6c49aa8be606509002898e73091ff2a3bbfc59c2c6c71d4660609f63aa92f98e3 + languageName: node + linkType: hard + "minimatch@npm:^8.0.2": version: 8.0.4 resolution: "minimatch@npm:8.0.4" @@ -17377,6 +17465,15 @@ __metadata: languageName: node linkType: hard +"redis-info@npm:^3.1.0": + version: 3.1.0 + resolution: "redis-info@npm:3.1.0" + dependencies: + lodash: "npm:^4.17.11" + checksum: 10c0/ec0f31d97893c5828cec7166486d74198c92160c60073b6f2fe805cdf575a10ddcccc7641737d44b8f451355f0ab5b6c7b0d79e8fc24742b75dd625f91ffee38 + languageName: node + linkType: hard + "redis-parser@npm:^3.0.0": version: 3.0.0 resolution: "redis-parser@npm:3.0.0" From 963f6fd69ef30ccd8a44883aa36eb8253f13e65b Mon Sep 17 00:00:00 2001 From: bkellam Date: Fri, 10 Oct 2025 12:32:34 -0700 Subject: [PATCH 02/24] add indexing --- packages/backend/src/index.ts | 4 +- packages/backend/src/indexSyncer.ts | 225 ++++++++++++++++++---------- 2 files changed, 149 insertions(+), 80 deletions(-) diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 3850d07b..411de187 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -72,9 +72,9 @@ const connectionManager = new ConnectionManager(prisma, settings, redis); const repoManager = new RepoManager(prisma, settings, redis, promClient, context); const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); const userPermissionSyncer = new UserPermissionSyncer(prisma, settings, redis); -const indexSyncer = new IndexSyncer(prisma, settings, redis); +const indexSyncer = new IndexSyncer(prisma, settings, redis, context); -await repoManager.validateIndexedReposHaveShards(); +// await repoManager.validateIndexedReposHaveShards(); connectionManager.startScheduler(); // repoManager.startScheduler(); diff --git a/packages/backend/src/indexSyncer.ts b/packages/backend/src/indexSyncer.ts index 11a88009..aba95fbf 100644 --- a/packages/backend/src/indexSyncer.ts +++ b/packages/backend/src/indexSyncer.ts @@ -4,9 +4,13 @@ import * as Sentry from '@sentry/node'; import { PrismaClient, Repo, RepoIndexingJobStatus } from "@sourcebot/db"; import { createLogger } from "@sourcebot/logger"; import express from 'express'; -import { BullBoardGroupMQAdapter, Queue, Worker } from "groupmq"; +import { BullBoardGroupMQAdapter, Job, Queue, ReservedJob, Worker } from "groupmq"; import { Redis } from 'ioredis'; -import { Settings } from "./types.js"; +import { AppContext, repoMetadataSchema, RepoWithConnections, Settings } from "./types.js"; +import { getAuthCredentialsForRepo, getRepoPath, measure } from './utils.js'; +import { existsSync } from 'fs'; +import { cloneRepository, fetchRepository, unsetGitConfig, upsertGitConfig } from './git.js'; +import { indexGitRepository } from './zoekt.js'; const logger = createLogger('index-syncer'); @@ -14,7 +18,7 @@ type IndexSyncJob = { jobId: string; } -const JOB_TIMEOUT_MS = 1000 * 60; // 60 second timeout. +const JOB_TIMEOUT_MS = 1000 * 60 * 60 * 6; // 6 hour indexing timeout export class IndexSyncer { private interval?: NodeJS.Timeout; @@ -25,6 +29,7 @@ export class IndexSyncer { private db: PrismaClient, private settings: Settings, redis: Redis, + private ctx: AppContext, ) { this.queue = new Queue({ redis, @@ -37,88 +42,19 @@ export class IndexSyncer { queue: this.queue, maxStalledCount: 1, stalledInterval: 1000, - handler: async (job) => { - const id = job.data.jobId; - const { repo } = await this.db.repoIndexingJob.update({ - where: { - id, - }, - data: { - status: RepoIndexingJobStatus.IN_PROGRESS, - }, - select: { - repo: true, - } - }); - - logger.info(`Running index job ${id} for repo ${repo.name}`); - - await new Promise(resolve => setTimeout(resolve, 1000 * 10)); - - return true; - }, - concurrency: 4, - }); - - this.worker.on('completed', async (job) => { - const { repo } = await this.db.repoIndexingJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - repo: { - update: { - indexedAt: new Date(), - } - }, - completedAt: new Date(), - }, - select: { - repo: true, - } - }); - - logger.info(`Completed index job ${job.data.jobId} for repo ${repo.name}`); - }); - - this.worker.on('failed', async (job) => { - const { repo } = await this.db.repoIndexingJob.update({ - where: { - id: job.data.jobId, - }, - data: { - status: RepoIndexingJobStatus.FAILED, - completedAt: new Date(), - errorMessage: job.failedReason, - }, - select: { - repo: true, - } - }); - - logger.error(`Failed index job ${job.data.jobId} for repo ${repo.name}`); - }); - - this.worker.on('stalled', async (jobId) => { - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: jobId }, - data: { - status: RepoIndexingJobStatus.FAILED, - completedAt: new Date(), - errorMessage: 'Job stalled', - }, - select: { repo: true } - }); - - logger.warn(`Job ${jobId} stalled for repo ${repo.name}`); + handler: this.runJob.bind(this), + concurrency: this.settings.maxRepoIndexingJobConcurrency, }); + this.worker.on('completed', this.onJobCompleted.bind(this)); + this.worker.on('failed', this.onJobFailed.bind(this)); + this.worker.on('stalled', this.onJobStalled.bind(this)); this.worker.on('error', async (error) => { Sentry.captureException(error); logger.error(`Index syncer worker error.`, error); }); + // @nocheckin const app = express(); const serverAdapter = new ExpressAdapter(); @@ -215,6 +151,139 @@ export class IndexSyncer { } } + private async runJob(job: ReservedJob) { + const id = job.data.jobId; + const { repo } = await this.db.repoIndexingJob.update({ + where: { + id, + }, + data: { + status: RepoIndexingJobStatus.IN_PROGRESS, + }, + select: { + repo: { + include: { + connections: { + include: { + connection: true, + } + } + } + } + } + }); + + await this.syncGitRepository(repo); + } + + private async syncGitRepository(repo: RepoWithConnections) { + const { path: repoPath, isReadOnly } = getRepoPath(repo, this.ctx); + + const metadata = repoMetadataSchema.parse(repo.metadata); + + const credentials = await getAuthCredentialsForRepo(repo, this.db); + const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; + const authHeader = credentials?.authHeader ?? undefined; + + if (existsSync(repoPath) && !isReadOnly) { + // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ + // write the clone URL (which could contain a auth token) to the + // `remote.origin.url` entry. For the upgrade scenario, we want + // to unset this key since it is no longer needed, hence this line. + // This will no-op if the key is already unset. + // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 + await unsetGitConfig(repoPath, ["remote.origin.url"]); + + logger.info(`Fetching ${repo.displayName}...`); + const { durationMs } = await measure(() => fetchRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`) + } + })); + const fetchDuration_s = durationMs / 1000; + + process.stdout.write('\n'); + logger.info(`Fetched ${repo.displayName} in ${fetchDuration_s}s`); + + } else if (!isReadOnly) { + logger.info(`Cloning ${repo.displayName}...`); + + const { durationMs } = await measure(() => cloneRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`) + } + })); + const cloneDuration_s = durationMs / 1000; + + process.stdout.write('\n'); + logger.info(`Cloned ${repo.displayName} in ${cloneDuration_s}s`); + } + + // Regardless of clone or fetch, always upsert the git config for the repo. + // This ensures that the git config is always up to date for whatever we + // have in the DB. + if (metadata.gitConfig && !isReadOnly) { + await upsertGitConfig(repoPath, metadata.gitConfig); + } + + logger.info(`Indexing ${repo.displayName}...`); + const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx)); + const indexDuration_s = durationMs / 1000; + logger.info(`Indexed ${repo.displayName} in ${indexDuration_s}s`); + } + + private async onJobCompleted(job: Job) { + const { repo } = await this.db.repoIndexingJob.update({ + where: { id: job.data.jobId }, + data: { + status: RepoIndexingJobStatus.COMPLETED, + repo: { + update: { + indexedAt: new Date(), + } + }, + completedAt: new Date(), + }, + select: { repo: true } + }); + + logger.info(`Completed index job ${job.data.jobId} for repo ${repo.name}`); + } + + private async onJobFailed(job: Job) { + const { repo } = await this.db.repoIndexingJob.update({ + where: { id: job.data.jobId }, + data: { + status: RepoIndexingJobStatus.FAILED, + completedAt: new Date(), + errorMessage: job.failedReason, + }, + select: { repo: true} + }); + + logger.error(`Failed index job ${job.data.jobId} for repo ${repo.name}`); + } + + private async onJobStalled(jobId: string) { + const { repo } = await this.db.repoIndexingJob.update({ + where: { id: jobId }, + data: { + status: RepoIndexingJobStatus.FAILED, + completedAt: new Date(), + errorMessage: 'Job stalled', + }, + select: { repo: true } + }); + + logger.error(`Job ${jobId} stalled for repo ${repo.name}`); + } + public dispose() { if (this.interval) { clearInterval(this.interval); From 775b87a06cfc1cdd9715f7bbb8fca468863ebb97 Mon Sep 17 00:00:00 2001 From: bkellam Date: Tue, 14 Oct 2025 20:16:43 -0700 Subject: [PATCH 03/24] further wip --- package.json | 6 +- packages/backend/src/connectionManager.ts | 6 +- .../backend/src/ee/repoPermissionSyncer.ts | 6 +- .../backend/src/ee/userPermissionSyncer.ts | 6 +- packages/backend/src/index.ts | 29 +- packages/backend/src/indexSyncer.ts | 353 ++++++++++++------ packages/backend/src/promClient.ts | 9 +- packages/backend/src/repoManager.ts | 8 +- packages/db/prisma/schema.prisma | 16 +- packages/logger/src/index.ts | 10 +- yarn.lock | 113 +++++- 11 files changed, 413 insertions(+), 149 deletions(-) diff --git a/package.json b/package.json index c5909e76..7c726c7e 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "cross-env SKIP_ENV_VALIDATION=1 yarn workspaces foreach -A run build", "test": "yarn workspaces foreach -A run test", - "dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web watch:mcp watch:schemas", + "dev": "concurrently --kill-others --names \"zoekt,worker,web,mcp,schemas\" 'yarn dev:zoekt' 'yarn dev:backend' 'yarn dev:web' 'yarn watch:mcp' 'yarn watch:schemas'", "with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --", "dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc", "dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch", @@ -21,9 +21,9 @@ "build:deps": "yarn workspaces foreach -R --from '{@sourcebot/schemas,@sourcebot/error,@sourcebot/crypto,@sourcebot/db,@sourcebot/shared}' run build" }, "devDependencies": { + "concurrently": "^9.2.1", "cross-env": "^7.0.3", - "dotenv-cli": "^8.0.0", - "npm-run-all": "^4.1.5" + "dotenv-cli": "^8.0.0" }, "packageManager": "yarn@4.7.0", "resolutions": { diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index e17fce06..ce023fc5 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -364,12 +364,12 @@ export class ConnectionManager { } } - public dispose() { + public async dispose() { if (this.interval) { clearInterval(this.interval); } - this.worker.close(); - this.queue.close(); + await this.worker.close(); + await this.queue.close(); } } diff --git a/packages/backend/src/ee/repoPermissionSyncer.ts b/packages/backend/src/ee/repoPermissionSyncer.ts index f411c3e3..453b94f6 100644 --- a/packages/backend/src/ee/repoPermissionSyncer.ts +++ b/packages/backend/src/ee/repoPermissionSyncer.ts @@ -101,12 +101,12 @@ export class RepoPermissionSyncer { }, 1000 * 5); } - public dispose() { + public async dispose() { if (this.interval) { clearInterval(this.interval); } - this.worker.close(); - this.queue.close(); + await this.worker.close(); + await this.queue.close(); } private async schedulePermissionSync(repos: Repo[]) { diff --git a/packages/backend/src/ee/userPermissionSyncer.ts b/packages/backend/src/ee/userPermissionSyncer.ts index 90ae8629..6ef77bcf 100644 --- a/packages/backend/src/ee/userPermissionSyncer.ts +++ b/packages/backend/src/ee/userPermissionSyncer.ts @@ -101,12 +101,12 @@ export class UserPermissionSyncer { }, 1000 * 5); } - public dispose() { + public async dispose() { if (this.interval) { clearInterval(this.interval); } - this.worker.close(); - this.queue.close(); + await this.worker.close(); + await this.queue.close(); } private async schedulePermissionSync(users: User[]) { diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 411de187..0b9006b9 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -90,13 +90,28 @@ else if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && hasEntitlement( } const cleanup = async (signal: string) => { - logger.info(`Recieved ${signal}, cleaning up...`); - - connectionManager.dispose(); - repoManager.dispose(); - repoPermissionSyncer.dispose(); - userPermissionSyncer.dispose(); - indexSyncer.dispose(); + logger.info(`Received ${signal}, cleaning up...`); + + const shutdownTimeout = 30000; // 30 seconds + + try { + await Promise.race([ + Promise.all([ + indexSyncer.dispose(), + repoManager.dispose(), + connectionManager.dispose(), + repoPermissionSyncer.dispose(), + userPermissionSyncer.dispose(), + promClient.dispose(), + ]), + new Promise((_, reject) => + setTimeout(() => reject(new Error('Shutdown timeout')), shutdownTimeout) + ) + ]); + logger.info('All workers shut down gracefully'); + } catch (error) { + logger.warn('Shutdown timeout or error, forcing exit:', error instanceof Error ? error.message : String(error)); + } await prisma.$disconnect(); await redis.quit(); diff --git a/packages/backend/src/indexSyncer.ts b/packages/backend/src/indexSyncer.ts index aba95fbf..19b20a77 100644 --- a/packages/backend/src/indexSyncer.ts +++ b/packages/backend/src/indexSyncer.ts @@ -1,29 +1,46 @@ import { createBullBoard } from '@bull-board/api'; import { ExpressAdapter } from '@bull-board/express'; import * as Sentry from '@sentry/node'; -import { PrismaClient, Repo, RepoIndexingJobStatus } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; +import { PrismaClient, Repo, RepoJobStatus, RepoJobType } from "@sourcebot/db"; +import { createLogger, Logger } from "@sourcebot/logger"; import express from 'express'; import { BullBoardGroupMQAdapter, Job, Queue, ReservedJob, Worker } from "groupmq"; import { Redis } from 'ioredis'; import { AppContext, repoMetadataSchema, RepoWithConnections, Settings } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoPath, measure } from './utils.js'; +import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, measure } from './utils.js'; import { existsSync } from 'fs'; -import { cloneRepository, fetchRepository, unsetGitConfig, upsertGitConfig } from './git.js'; +import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js'; import { indexGitRepository } from './zoekt.js'; +import { rm, readdir } from 'fs/promises'; -const logger = createLogger('index-syncer'); +const LOG_TAG = 'index-syncer'; +const logger = createLogger(LOG_TAG); +const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); -type IndexSyncJob = { +type JobPayload = { + type: 'INDEX' | 'CLEANUP'; jobId: string; -} + repoId: number; + repoName: string; +}; const JOB_TIMEOUT_MS = 1000 * 60 * 60 * 6; // 6 hour indexing timeout + +const groupmqLifecycleExceptionWrapper = async (name: string, fn: () => Promise) => { + try { + await fn(); + } catch (error) { + Sentry.captureException(error); + logger.error(`Exception thrown while executing lifecycle function \`${name}\`.`, error); + } +} + + export class IndexSyncer { private interval?: NodeJS.Timeout; - private queue: Queue; - private worker: Worker; + private queue: Queue; + private worker: Worker; constructor( private db: PrismaClient, @@ -31,28 +48,26 @@ export class IndexSyncer { redis: Redis, private ctx: AppContext, ) { - this.queue = new Queue({ + this.queue = new Queue({ redis, namespace: 'index-sync-queue', jobTimeoutMs: JOB_TIMEOUT_MS, - // logger: true, + logger, + maxAttempts: 1, }); - this.worker = new Worker({ + this.worker = new Worker({ queue: this.queue, maxStalledCount: 1, - stalledInterval: 1000, handler: this.runJob.bind(this), concurrency: this.settings.maxRepoIndexingJobConcurrency, + logger, }); this.worker.on('completed', this.onJobCompleted.bind(this)); this.worker.on('failed', this.onJobFailed.bind(this)); this.worker.on('stalled', this.onJobStalled.bind(this)); - this.worker.on('error', async (error) => { - Sentry.captureException(error); - logger.error(`Index syncer worker error.`, error); - }); + this.worker.on('error', this.onWorkerError.bind(this)); // @nocheckin const app = express(); @@ -69,75 +84,133 @@ export class IndexSyncer { public async startScheduler() { this.interval = setInterval(async () => { - const thresholdDate = new Date(Date.now() - this.settings.reindexIntervalMs); - - const repos = await this.db.repo.findMany({ - where: { - AND: [ - { - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } }, - ] - }, - { - NOT: { - indexingJobs: { - some: { - OR: [ - // Don't schedule if there are active jobs that were created within the threshold date. - // This handles the case where a job is stuck in a pending state and will never be scheduled. - { - AND: [ - { - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] + await this.scheduleIndexJobs(); + await this.scheduleCleanupJobs(); + }, 1000 * 5); + + this.worker.run(); + } + + private async scheduleIndexJobs() { + const thresholdDate = new Date(Date.now() - this.settings.reindexIntervalMs); + const reposToIndex = await this.db.repo.findMany({ + where: { + AND: [ + { + OR: [ + { indexedAt: null }, + { indexedAt: { lt: thresholdDate } }, + ] + }, + { + NOT: { + jobs: { + some: { + AND: [ + { + type: RepoJobType.INDEX, + }, + { + OR: [ + // Don't schedule if there are active jobs that were created within the threshold date. + // This handles the case where a job is stuck in a pending state and will never be scheduled. + { + AND: [ + { + status: { + in: [ + RepoJobStatus.PENDING, + RepoJobStatus.IN_PROGRESS, + ] + }, }, - }, - { - createdAt: { - gt: thresholdDate, + { + createdAt: { + gt: thresholdDate, + } } - } - ] - }, - // Don't schedule if there are recent failed jobs (within the threshold date). - { - AND: [ - { status: RepoIndexingJobStatus.FAILED }, - { completedAt: { gt: thresholdDate } }, - ] - } + ] + }, + // Don't schedule if there are recent failed jobs (within the threshold date). + { + AND: [ + { status: RepoJobStatus.FAILED }, + { completedAt: { gt: thresholdDate } }, + ] + } + ] + } + ] + } + } + } + } + ], + } + }); + + if (reposToIndex.length > 0) { + await this.createJobs(reposToIndex, RepoJobType.INDEX); + } + } + + private async scheduleCleanupJobs() { + const thresholdDate = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs); + + const reposToCleanup = await this.db.repo.findMany({ + where: { + connections: { + none: {} + }, + OR: [ + { indexedAt: null }, + { indexedAt: { lt: thresholdDate } }, + ], + // Don't schedule if there are active jobs that were created within the threshold date. + NOT: { + jobs: { + some: { + AND: [ + { + type: RepoJobType.CLEANUP, + }, + { + status: { + in: [ + RepoJobStatus.PENDING, + RepoJobStatus.IN_PROGRESS, ] + }, + }, + { + createdAt: { + gt: thresholdDate, } } - } + ] } - ], + } } - }); - - if (repos.length === 0) { - return; } + }); - await this.scheduleIndexSync(repos); - }, 1000 * 5); - - this.worker.run(); + if (reposToCleanup.length > 0) { + await this.createJobs(reposToCleanup, RepoJobType.CLEANUP); + } } - private async scheduleIndexSync(repos: Repo[]) { + private async createJobs(repos: Repo[], type: RepoJobType) { // @note: we don't perform this in a transaction because // we want to avoid the situation where a job is created and run // prior to the transaction being committed. - const jobs = await this.db.repoIndexingJob.createManyAndReturn({ + const jobs = await this.db.repoJob.createManyAndReturn({ data: repos.map(repo => ({ + type, repoId: repo.id, - })) + })), + include: { + repo: true, + } }); for (const job of jobs) { @@ -145,22 +218,29 @@ export class IndexSyncer { groupId: `repo:${job.repoId}`, data: { jobId: job.id, + type, + repoName: job.repo.name, + repoId: job.repo.id, }, jobId: job.id, }); } } - private async runJob(job: ReservedJob) { + private async runJob(job: ReservedJob) { const id = job.data.jobId; - const { repo } = await this.db.repoIndexingJob.update({ + const logger = createJobLogger(id); + logger.info(`Running job ${id} for repo ${job.data.repoName}`); + + const { repo, type: jobType } = await this.db.repoJob.update({ where: { id, }, data: { - status: RepoIndexingJobStatus.IN_PROGRESS, + status: RepoJobStatus.IN_PROGRESS, }, select: { + type: true, repo: { include: { connections: { @@ -173,10 +253,14 @@ export class IndexSyncer { } }); - await this.syncGitRepository(repo); + if (jobType === RepoJobType.INDEX) { + await this.indexRepository(repo, logger); + } else if (jobType === RepoJobType.CLEANUP) { + await this.cleanupRepository(repo, logger); + } } - private async syncGitRepository(repo: RepoWithConnections) { + private async indexRepository(repo: RepoWithConnections, logger: Logger) { const { path: repoPath, isReadOnly } = getRepoPath(repo, this.ctx); const metadata = repoMetadataSchema.parse(repo.metadata); @@ -185,6 +269,14 @@ export class IndexSyncer { const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; const authHeader = credentials?.authHeader ?? undefined; + // If the repo path exists but it is not a valid git repository root, this indicates + // that the repository is in a bad state. To fix, we remove the directory and perform + // a fresh clone. + if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot(repoPath)) && !isReadOnly) { + logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); + await rm(repoPath, { recursive: true, force: true }); + } + if (existsSync(repoPath) && !isReadOnly) { // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ // write the clone URL (which could contain a auth token) to the @@ -238,57 +330,94 @@ export class IndexSyncer { logger.info(`Indexed ${repo.displayName} in ${indexDuration_s}s`); } - private async onJobCompleted(job: Job) { - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - repo: { - update: { + private async cleanupRepository(repo: Repo, logger: Logger) { + const { path: repoPath, isReadOnly } = getRepoPath(repo, this.ctx); + if (existsSync(repoPath) && !isReadOnly) { + logger.info(`Deleting repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + const shardPrefix = getShardPrefix(repo.orgId, repo.id); + const files = (await readdir(this.ctx.indexPath)).filter(file => file.startsWith(shardPrefix)); + for (const file of files) { + const filePath = `${this.ctx.indexPath}/${file}`; + logger.info(`Deleting shard file ${filePath}`); + await rm(filePath, { force: true }); + } + } + + private onJobCompleted = async (job: Job) => + groupmqLifecycleExceptionWrapper('onJobCompleted', async () => { + const logger = createJobLogger(job.data.jobId); + const jobData = await this.db.repoJob.update({ + where: { id: job.data.jobId }, + data: { + status: RepoJobStatus.COMPLETED, + completedAt: new Date(), + } + }); + + if (jobData.type === RepoJobType.INDEX) { + const repo = await this.db.repo.update({ + where: { id: jobData.repoId }, + data: { indexedAt: new Date(), } - }, - completedAt: new Date(), - }, - select: { repo: true } + }); + + logger.info(`Completed index job ${job.data.jobId} for repo ${repo.name}`); + } + else if (jobData.type === RepoJobType.CLEANUP) { + const repo = await this.db.repo.delete({ + where: { id: jobData.repoId }, + }); + + logger.info(`Completed cleanup job ${job.data.jobId} for repo ${repo.name}`); + } }); - logger.info(`Completed index job ${job.data.jobId} for repo ${repo.name}`); - } + private onJobFailed = async (job: Job) => + groupmqLifecycleExceptionWrapper('onJobFailed', async () => { + const logger = createJobLogger(job.data.jobId); - private async onJobFailed(job: Job) { - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: job.data.jobId }, - data: { - status: RepoIndexingJobStatus.FAILED, - completedAt: new Date(), - errorMessage: job.failedReason, - }, - select: { repo: true} + const { repo } = await this.db.repoJob.update({ + where: { id: job.data.jobId }, + data: { + completedAt: new Date(), + errorMessage: job.failedReason, + }, + select: { repo: true } + }); + + logger.error(`Failed job ${job.data.jobId} for repo ${repo.name}`); }); - logger.error(`Failed index job ${job.data.jobId} for repo ${repo.name}`); - } + private onJobStalled = async (jobId: string) => + groupmqLifecycleExceptionWrapper('onJobStalled', async () => { + const logger = createJobLogger(jobId); + const { repo } = await this.db.repoJob.update({ + where: { id: jobId }, + data: { + status: RepoJobStatus.FAILED, + completedAt: new Date(), + errorMessage: 'Job stalled', + }, + select: { repo: true } + }); - private async onJobStalled(jobId: string) { - const { repo } = await this.db.repoIndexingJob.update({ - where: { id: jobId }, - data: { - status: RepoIndexingJobStatus.FAILED, - completedAt: new Date(), - errorMessage: 'Job stalled', - }, - select: { repo: true } + logger.error(`Job ${jobId} stalled for repo ${repo.name}`); }); - logger.error(`Job ${jobId} stalled for repo ${repo.name}`); + private async onWorkerError(error: Error) { + Sentry.captureException(error); + logger.error(`Index syncer worker error.`, error); } - public dispose() { + public async dispose() { if (this.interval) { clearInterval(this.interval); } - this.worker.close(); - this.queue.close(); + await this.worker.close(); + await this.queue.close(); } } \ No newline at end of file diff --git a/packages/backend/src/promClient.ts b/packages/backend/src/promClient.ts index 058cfe0b..4b806f6e 100644 --- a/packages/backend/src/promClient.ts +++ b/packages/backend/src/promClient.ts @@ -1,4 +1,5 @@ import express, { Request, Response } from 'express'; +import { Server } from 'http'; import client, { Registry, Counter, Gauge } from 'prom-client'; import { createLogger } from "@sourcebot/logger"; @@ -7,6 +8,8 @@ const logger = createLogger('prometheus-client'); export class PromClient { private registry: Registry; private app: express.Application; + private server: Server; + public activeRepoIndexingJobs: Gauge; public pendingRepoIndexingJobs: Gauge; public repoIndexingReattemptsTotal: Counter; @@ -98,12 +101,12 @@ export class PromClient { res.end(metrics); }); - this.app.listen(this.PORT, () => { + this.server = this.app.listen(this.PORT, () => { logger.info(`Prometheus metrics server is running on port ${this.PORT}`); }); } - getRegistry(): Registry { - return this.registry; + dispose() { + this.server.close(); } } \ No newline at end of file diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts index 89e41673..33f8bdad 100644 --- a/packages/backend/src/repoManager.ts +++ b/packages/backend/src/repoManager.ts @@ -558,9 +558,9 @@ export class RepoManager { if (this.interval) { clearInterval(this.interval); } - this.indexWorker.close(); - this.indexQueue.close(); - this.gcQueue.close(); - this.gcWorker.close(); + await this.indexWorker.close(); + await this.indexQueue.close(); + await this.gcQueue.close(); + await this.gcWorker.close(); } } \ No newline at end of file diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 75e30ec9..16ed94f0 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -54,13 +54,15 @@ model Repo { webUrl String? connections RepoToConnection[] imageUrl String? + + /// @deprecated status tracking is now done via the `jobs` table. repoIndexingStatus RepoIndexingStatus @default(NEW) permittedUsers UserToRepoPermission[] permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. - indexingJobs RepoIndexingJob[] + jobs RepoJob[] indexedAt DateTime? /// When the repo was last indexed successfully. external_id String /// The id of the repo in the external service @@ -76,16 +78,22 @@ model Repo { @@index([orgId]) } -enum RepoIndexingJobStatus { +enum RepoJobStatus { PENDING IN_PROGRESS COMPLETED FAILED } -model RepoIndexingJob { +enum RepoJobType { + INDEX + CLEANUP +} + +model RepoJob { id String @id @default(cuid()) - status RepoIndexingJobStatus @default(PENDING) + type RepoJobType + status RepoJobStatus @default(PENDING) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt completedAt DateTime? diff --git a/packages/logger/src/index.ts b/packages/logger/src/index.ts index d3998d2c..635c8b3c 100644 --- a/packages/logger/src/index.ts +++ b/packages/logger/src/index.ts @@ -1,4 +1,4 @@ -import winston, { format } from 'winston'; +import winston, { format, Logger } from 'winston'; import { Logtail } from '@logtail/node'; import { LogtailTransport } from '@logtail/winston'; import { MESSAGE } from 'triple-beam'; @@ -48,7 +48,7 @@ const createLogger = (label: string) => { format: combine( errors({ stack: true }), timestamp(), - labelFn({ label: label }) + labelFn({ label: label }), ), transports: [ new winston.transports.Console({ @@ -84,4 +84,8 @@ const createLogger = (label: string) => { export { createLogger -}; \ No newline at end of file +}; + +export type { + Logger, +} \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 6fe84282..34be0b97 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10061,6 +10061,17 @@ __metadata: languageName: node linkType: hard +"cliui@npm:^8.0.1": + version: 8.0.1 + resolution: "cliui@npm:8.0.1" + dependencies: + string-width: "npm:^4.2.0" + strip-ansi: "npm:^6.0.1" + wrap-ansi: "npm:^7.0.0" + checksum: 10c0/4bda0f09c340cbb6dfdc1ed508b3ca080f12992c18d68c6be4d9cf51756033d5266e61ec57529e610dacbf4da1c634423b0c1b11037709cc6b09045cbd815df5 + languageName: node + linkType: hard + "clone@npm:^1.0.2": version: 1.0.4 resolution: "clone@npm:1.0.4" @@ -10459,6 +10470,23 @@ __metadata: languageName: node linkType: hard +"concurrently@npm:^9.2.1": + version: 9.2.1 + resolution: "concurrently@npm:9.2.1" + dependencies: + chalk: "npm:4.1.2" + rxjs: "npm:7.8.2" + shell-quote: "npm:1.8.3" + supports-color: "npm:8.1.1" + tree-kill: "npm:1.2.2" + yargs: "npm:17.7.2" + bin: + conc: dist/bin/concurrently.js + concurrently: dist/bin/concurrently.js + checksum: 10c0/da37f239f82eb7ac24f5ddb56259861e5f1d6da2ade7602b6ea7ad3101b13b5ccec02a77b7001402d1028ff2fdc38eed55644b32853ad5abf30e057002a963aa + languageName: node + linkType: hard + "content-disposition@npm:0.5.4": version: 0.5.4 resolution: "content-disposition@npm:0.5.4" @@ -11810,7 +11838,7 @@ __metadata: languageName: node linkType: hard -"escalade@npm:^3.2.0": +"escalade@npm:^3.1.1, escalade@npm:^3.2.0": version: 3.2.0 resolution: "escalade@npm:3.2.0" checksum: 10c0/ced4dd3a78e15897ed3be74e635110bbf3b08877b0a41be50dcb325ee0e0b5f65fc2d50e9845194d7c4633f327e2e1c6cce00a71b617c5673df0374201d67f65 @@ -12781,6 +12809,13 @@ __metadata: languageName: node linkType: hard +"get-caller-file@npm:^2.0.5": + version: 2.0.5 + resolution: "get-caller-file@npm:2.0.5" + checksum: 10c0/c6c7b60271931fa752aeb92f2b47e355eac1af3a2673f47c9589e8f8a41adc74d45551c1bc57b5e66a80609f10ffb72b6f575e4370d61cc3f7f3aaff01757cde + languageName: node + linkType: hard + "get-intrinsic@npm:^1.2.4, get-intrinsic@npm:^1.2.5, get-intrinsic@npm:^1.2.6, get-intrinsic@npm:^1.2.7, get-intrinsic@npm:^1.3.0": version: 1.3.0 resolution: "get-intrinsic@npm:1.3.0" @@ -17617,6 +17652,13 @@ __metadata: languageName: node linkType: hard +"require-directory@npm:^2.1.1": + version: 2.1.1 + resolution: "require-directory@npm:2.1.1" + checksum: 10c0/83aa76a7bc1531f68d92c75a2ca2f54f1b01463cb566cf3fbc787d0de8be30c9dbc211d1d46be3497dac5785fe296f2dd11d531945ac29730643357978966e99 + languageName: node + linkType: hard + "require-from-string@npm:^2.0.2": version: 2.0.2 resolution: "require-from-string@npm:2.0.2" @@ -17931,9 +17973,9 @@ __metadata: version: 0.0.0-use.local resolution: "root-workspace-0b6124@workspace:." dependencies: + concurrently: "npm:^9.2.1" cross-env: "npm:^7.0.3" dotenv-cli: "npm:^8.0.0" - npm-run-all: "npm:^4.1.5" languageName: unknown linkType: soft @@ -18015,6 +18057,15 @@ __metadata: languageName: node linkType: hard +"rxjs@npm:7.8.2": + version: 7.8.2 + resolution: "rxjs@npm:7.8.2" + dependencies: + tslib: "npm:^2.1.0" + checksum: 10c0/1fcd33d2066ada98ba8f21fcbbcaee9f0b271de1d38dc7f4e256bfbc6ffcdde68c8bfb69093de7eeb46f24b1fb820620bf0223706cff26b4ab99a7ff7b2e2c45 + languageName: node + linkType: hard + "safe-array-concat@npm:^1.1.3": version: 1.1.3 resolution: "safe-array-concat@npm:1.1.3" @@ -18443,6 +18494,13 @@ __metadata: languageName: node linkType: hard +"shell-quote@npm:1.8.3": + version: 1.8.3 + resolution: "shell-quote@npm:1.8.3" + checksum: 10c0/bee87c34e1e986cfb4c30846b8e6327d18874f10b535699866f368ade11ea4ee45433d97bf5eada22c4320c27df79c3a6a7eb1bf3ecfc47f2c997d9e5e2672fd + languageName: node + linkType: hard + "shell-quote@npm:^1.6.1": version: 1.8.2 resolution: "shell-quote@npm:1.8.2" @@ -18864,7 +18922,7 @@ __metadata: languageName: node linkType: hard -"string-width-cjs@npm:string-width@^4.2.0, string-width@npm:^4.1.0": +"string-width-cjs@npm:string-width@^4.2.0, string-width@npm:^4.1.0, string-width@npm:^4.2.0, string-width@npm:^4.2.3": version: 4.2.3 resolution: "string-width@npm:4.2.3" dependencies: @@ -19128,6 +19186,15 @@ __metadata: languageName: node linkType: hard +"supports-color@npm:8.1.1": + version: 8.1.1 + resolution: "supports-color@npm:8.1.1" + dependencies: + has-flag: "npm:^4.0.0" + checksum: 10c0/ea1d3c275dd604c974670f63943ed9bd83623edc102430c05adb8efc56ba492746b6e95386e7831b872ec3807fd89dd8eb43f735195f37b5ec343e4234cc7e89 + languageName: node + linkType: hard + "supports-color@npm:^5.3.0, supports-color@npm:^5.5.0": version: 5.5.0 resolution: "supports-color@npm:5.5.0" @@ -19438,6 +19505,15 @@ __metadata: languageName: node linkType: hard +"tree-kill@npm:1.2.2": + version: 1.2.2 + resolution: "tree-kill@npm:1.2.2" + bin: + tree-kill: cli.js + checksum: 10c0/7b1b7c7f17608a8f8d20a162e7957ac1ef6cd1636db1aba92f4e072dc31818c2ff0efac1e3d91064ede67ed5dc57c565420531a8134090a12ac10cf792ab14d2 + languageName: node + linkType: hard + "trim-lines@npm:^3.0.0": version: 3.0.1 resolution: "trim-lines@npm:3.0.1" @@ -20487,7 +20563,7 @@ __metadata: languageName: node linkType: hard -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0, wrap-ansi@npm:^7.0.0": version: 7.0.0 resolution: "wrap-ansi@npm:7.0.0" dependencies: @@ -20574,6 +20650,13 @@ __metadata: languageName: node linkType: hard +"y18n@npm:^5.0.5": + version: 5.0.8 + resolution: "y18n@npm:5.0.8" + checksum: 10c0/4df2842c36e468590c3691c894bc9cdbac41f520566e76e24f59401ba7d8b4811eb1e34524d57e54bc6d864bcb66baab7ffd9ca42bf1eda596618f9162b91249 + languageName: node + linkType: hard + "yallist@npm:^3.0.2": version: 3.1.1 resolution: "yallist@npm:3.1.1" @@ -20604,6 +20687,28 @@ __metadata: languageName: node linkType: hard +"yargs-parser@npm:^21.1.1": + version: 21.1.1 + resolution: "yargs-parser@npm:21.1.1" + checksum: 10c0/f84b5e48169479d2f402239c59f084cfd1c3acc197a05c59b98bab067452e6b3ea46d4dd8ba2985ba7b3d32a343d77df0debd6b343e5dae3da2aab2cdf5886b2 + languageName: node + linkType: hard + +"yargs@npm:17.7.2": + version: 17.7.2 + resolution: "yargs@npm:17.7.2" + dependencies: + cliui: "npm:^8.0.1" + escalade: "npm:^3.1.1" + get-caller-file: "npm:^2.0.5" + require-directory: "npm:^2.1.1" + string-width: "npm:^4.2.3" + y18n: "npm:^5.0.5" + yargs-parser: "npm:^21.1.1" + checksum: 10c0/ccd7e723e61ad5965fffbb791366db689572b80cca80e0f96aad968dfff4156cd7cd1ad18607afe1046d8241e6fb2d6c08bf7fa7bfb5eaec818735d8feac8f05 + languageName: node + linkType: hard + "yocto-queue@npm:^0.1.0": version: 0.1.0 resolution: "yocto-queue@npm:0.1.0" From 32c68e7b72d6ebd36e7b82aff15cace9465c5ac4 Mon Sep 17 00:00:00 2001 From: bkellam Date: Tue, 14 Oct 2025 21:24:04 -0700 Subject: [PATCH 04/24] wip --- packages/backend/src/indexSyncer.ts | 109 +++++++++++++--------------- packages/backend/src/utils.ts | 17 +++++ 2 files changed, 67 insertions(+), 59 deletions(-) diff --git a/packages/backend/src/indexSyncer.ts b/packages/backend/src/indexSyncer.ts index 19b20a77..6bf2ac0a 100644 --- a/packages/backend/src/indexSyncer.ts +++ b/packages/backend/src/indexSyncer.ts @@ -1,17 +1,15 @@ -import { createBullBoard } from '@bull-board/api'; -import { ExpressAdapter } from '@bull-board/express'; import * as Sentry from '@sentry/node'; import { PrismaClient, Repo, RepoJobStatus, RepoJobType } from "@sourcebot/db"; import { createLogger, Logger } from "@sourcebot/logger"; -import express from 'express'; -import { BullBoardGroupMQAdapter, Job, Queue, ReservedJob, Worker } from "groupmq"; -import { Redis } from 'ioredis'; -import { AppContext, repoMetadataSchema, RepoWithConnections, Settings } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, measure } from './utils.js'; import { existsSync } from 'fs'; +import { readdir, rm } from 'fs/promises'; +import { Job, Queue, ReservedJob, Worker } from "groupmq"; +import { Redis } from 'ioredis'; +import { env } from './env.js'; import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js'; +import { AppContext, repoMetadataSchema, RepoWithConnections, Settings } from "./types.js"; +import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js'; import { indexGitRepository } from './zoekt.js'; -import { rm, readdir } from 'fs/promises'; const LOG_TAG = 'index-syncer'; const logger = createLogger(LOG_TAG); @@ -26,17 +24,6 @@ type JobPayload = { const JOB_TIMEOUT_MS = 1000 * 60 * 60 * 6; // 6 hour indexing timeout - -const groupmqLifecycleExceptionWrapper = async (name: string, fn: () => Promise) => { - try { - await fn(); - } catch (error) { - Sentry.captureException(error); - logger.error(`Exception thrown while executing lifecycle function \`${name}\`.`, error); - } -} - - export class IndexSyncer { private interval?: NodeJS.Timeout; private queue: Queue; @@ -52,8 +39,10 @@ export class IndexSyncer { redis, namespace: 'index-sync-queue', jobTimeoutMs: JOB_TIMEOUT_MS, - logger, - maxAttempts: 1, + maxAttempts: 3, + ...(env.SOURCEBOT_LOG_LEVEL === 'debug' ? { + logger, + }: {}), }); this.worker = new Worker({ @@ -61,25 +50,15 @@ export class IndexSyncer { maxStalledCount: 1, handler: this.runJob.bind(this), concurrency: this.settings.maxRepoIndexingJobConcurrency, - logger, + ...(env.SOURCEBOT_LOG_LEVEL === 'debug' ? { + logger, + }: {}), }); this.worker.on('completed', this.onJobCompleted.bind(this)); this.worker.on('failed', this.onJobFailed.bind(this)); this.worker.on('stalled', this.onJobStalled.bind(this)); this.worker.on('error', this.onWorkerError.bind(this)); - - // @nocheckin - const app = express(); - const serverAdapter = new ExpressAdapter(); - - createBullBoard({ - queues: [new BullBoardGroupMQAdapter(this.queue, { displayName: 'Index Sync' })], - serverAdapter, - }); - - app.use('/', serverAdapter.getRouter()); - app.listen(3070); } public async startScheduler() { @@ -215,7 +194,7 @@ export class IndexSyncer { for (const job of jobs) { await this.queue.add({ - groupId: `repo:${job.repoId}`, + groupId: `repo:${job.repoId}_${job.repo.name}`, data: { jobId: job.id, type, @@ -230,7 +209,7 @@ export class IndexSyncer { private async runJob(job: ReservedJob) { const id = job.data.jobId; const logger = createJobLogger(id); - logger.info(`Running job ${id} for repo ${job.data.repoName}`); + logger.info(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId}) (attempt ${job.attempts + 1} / ${job.maxAttempts})`); const { repo, type: jobType } = await this.db.repoJob.update({ where: { @@ -286,35 +265,35 @@ export class IndexSyncer { // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 await unsetGitConfig(repoPath, ["remote.origin.url"]); - logger.info(`Fetching ${repo.displayName}...`); + logger.info(`Fetching ${repo.name} (id: ${repo.id})...`); const { durationMs } = await measure(() => fetchRepository({ cloneUrl: cloneUrlMaybeWithToken, authHeader, path: repoPath, onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`) + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) } })); const fetchDuration_s = durationMs / 1000; process.stdout.write('\n'); - logger.info(`Fetched ${repo.displayName} in ${fetchDuration_s}s`); + logger.info(`Fetched ${repo.name} (id: ${repo.id}) in ${fetchDuration_s}s`); } else if (!isReadOnly) { - logger.info(`Cloning ${repo.displayName}...`); + logger.info(`Cloning ${repo.name} (id: ${repo.id})...`); const { durationMs } = await measure(() => cloneRepository({ cloneUrl: cloneUrlMaybeWithToken, authHeader, path: repoPath, onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`) + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) } })); const cloneDuration_s = durationMs / 1000; process.stdout.write('\n'); - logger.info(`Cloned ${repo.displayName} in ${cloneDuration_s}s`); + logger.info(`Cloned ${repo.name} (id: ${repo.id}) in ${cloneDuration_s}s`); } // Regardless of clone or fetch, always upsert the git config for the repo. @@ -324,10 +303,10 @@ export class IndexSyncer { await upsertGitConfig(repoPath, metadata.gitConfig); } - logger.info(`Indexing ${repo.displayName}...`); + logger.info(`Indexing ${repo.name} (id: ${repo.id})...`); const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx)); const indexDuration_s = durationMs / 1000; - logger.info(`Indexed ${repo.displayName} in ${indexDuration_s}s`); + logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); } private async cleanupRepository(repo: Repo, logger: Logger) { @@ -347,7 +326,7 @@ export class IndexSyncer { } private onJobCompleted = async (job: Job) => - groupmqLifecycleExceptionWrapper('onJobCompleted', async () => { + groupmqLifecycleExceptionWrapper('onJobCompleted', logger, async () => { const logger = createJobLogger(job.data.jobId); const jobData = await this.db.repoJob.update({ where: { id: job.data.jobId }, @@ -365,35 +344,47 @@ export class IndexSyncer { } }); - logger.info(`Completed index job ${job.data.jobId} for repo ${repo.name}`); + logger.info(`Completed index job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); } else if (jobData.type === RepoJobType.CLEANUP) { const repo = await this.db.repo.delete({ where: { id: jobData.repoId }, }); - logger.info(`Completed cleanup job ${job.data.jobId} for repo ${repo.name}`); + logger.info(`Completed cleanup job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); } }); private onJobFailed = async (job: Job) => - groupmqLifecycleExceptionWrapper('onJobFailed', async () => { + groupmqLifecycleExceptionWrapper('onJobFailed', logger, async () => { const logger = createJobLogger(job.data.jobId); - const { repo } = await this.db.repoJob.update({ - where: { id: job.data.jobId }, - data: { - completedAt: new Date(), - errorMessage: job.failedReason, - }, - select: { repo: true } - }); + const attempt = job.attemptsMade + 1; + const wasLastAttempt = attempt >= job.opts.attempts; - logger.error(`Failed job ${job.data.jobId} for repo ${repo.name}`); + if (wasLastAttempt) { + const { repo } = await this.db.repoJob.update({ + where: { id: job.data.jobId }, + data: { + status: RepoJobStatus.FAILED, + completedAt: new Date(), + errorMessage: job.failedReason, + }, + select: { repo: true } + }); + + logger.error(`Failed job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id}). Attempt ${attempt} / ${job.opts.attempts}. Failing job.`); + } else { + const repo = await this.db.repo.findUniqueOrThrow({ + where: { id: job.data.repoId }, + }); + + logger.warn(`Failed job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id}). Attempt ${attempt} / ${job.opts.attempts}. Retrying.`); + } }); private onJobStalled = async (jobId: string) => - groupmqLifecycleExceptionWrapper('onJobStalled', async () => { + groupmqLifecycleExceptionWrapper('onJobStalled', logger, async () => { const logger = createJobLogger(jobId); const { repo } = await this.db.repoJob.update({ where: { id: jobId }, @@ -405,7 +396,7 @@ export class IndexSyncer { select: { repo: true } }); - logger.error(`Job ${jobId} stalled for repo ${repo.name}`); + logger.error(`Job ${jobId} stalled for repo ${repo.name} (id: ${repo.id})`); }); private async onWorkerError(error: Error) { diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index e6ac5f93..f081ce1b 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -241,3 +241,20 @@ const createGitCloneUrlWithToken = (cloneUrl: string, credentials: { username?: } return url.toString(); } + + +/** + * Wraps groupmq worker lifecycle callbacks with exception handling. This prevents + * uncaught exceptions (e.g., like a RepoJob not existing in the DB) from crashing + * the app. + * @see: https://openpanel-dev.github.io/groupmq/api-worker/#events + */ +export const groupmqLifecycleExceptionWrapper = async (name: string, logger: Logger, fn: () => Promise) => { + try { + await fn(); + } catch (error) { + Sentry.captureException(error); + logger.error(`Exception thrown while executing lifecycle function \`${name}\`.`, error); + } +} + From d315292fe7fd1c1b2c00240263fbe3560fcf81c4 Mon Sep 17 00:00:00 2001 From: bkellam Date: Tue, 14 Oct 2025 22:52:15 -0700 Subject: [PATCH 05/24] wip --- packages/backend/src/constants.ts | 7 ++- packages/backend/src/env.ts | 1 + packages/backend/src/git.ts | 77 ++++++++++++++++++----------- packages/backend/src/index.ts | 25 +++------- packages/backend/src/indexSyncer.ts | 23 +++++---- packages/backend/src/repoManager.ts | 18 +++---- packages/backend/src/types.ts | 14 ------ packages/backend/src/utils.ts | 7 +-- packages/backend/src/zoekt.ts | 18 +++---- 9 files changed, 96 insertions(+), 94 deletions(-) diff --git a/packages/backend/src/constants.ts b/packages/backend/src/constants.ts index 89778fb2..3ede6d4e 100644 --- a/packages/backend/src/constants.ts +++ b/packages/backend/src/constants.ts @@ -1,4 +1,6 @@ +import { env } from "./env.js"; import { Settings } from "./types.js"; +import path from "path"; /** * Default settings. @@ -22,4 +24,7 @@ export const DEFAULT_SETTINGS: Settings = { export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [ 'github', -]; \ No newline at end of file +]; + +export const REPOS_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'repos'); +export const INDEX_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'index'); \ No newline at end of file diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 80bbba5e..5e6e1e1d 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -44,6 +44,7 @@ export const env = createEnv({ LOGTAIL_TOKEN: z.string().optional(), LOGTAIL_HOST: z.string().url().optional(), SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"), + DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'), DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"), CONFIG_PATH: z.string().optional(), diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts index c1110625..e78b7db6 100644 --- a/packages/backend/src/git.ts +++ b/packages/backend/src/git.ts @@ -1,9 +1,47 @@ import { CheckRepoActions, GitConfigScope, simpleGit, SimpleGitProgressEvent } from 'simple-git'; import { mkdir } from 'node:fs/promises'; import { env } from './env.js'; +import { dirname, resolve } from 'node:path'; +import { existsSync } from 'node:fs'; type onProgressFn = (event: SimpleGitProgressEvent) => void; +/** + * Creates a simple-git client that has it's working directory + * set to the given path. + */ +const createGitClientForPath = (path: string, onProgress?: onProgressFn) => { + if (!existsSync(path)) { + throw new Error(`Path ${path} does not exist`); + } + + const parentPath = resolve(dirname(path)); + + const git = simpleGit({ + progress: onProgress, + }) + .env({ + ...process.env, + /** + * @note on some inside-baseball on why this is necessary: The specific + * issue we saw was that a `git clone` would fail without throwing, and + * then a subsequent `git config` command would run, but since the clone + * failed, it wouldn't be running in a git directory. Git would then walk + * up the directory tree until it either found a git directory (in the case + * of the development env) or it would hit a GIT_DISCOVERY_ACROSS_FILESYSTEM + * error when trying to cross a filesystem boundary (in the prod case). + * GIT_CEILING_DIRECTORIES ensures that this walk will be limited to the + * parent directory. + */ + GIT_CEILING_DIRECTORIES: parentPath, + }) + .cwd({ + path, + }); + + return git; +} + export const cloneRepository = async ( { cloneUrl, @@ -20,11 +58,7 @@ export const cloneRepository = async ( try { await mkdir(path, { recursive: true }); - const git = simpleGit({ - progress: onProgress, - }).cwd({ - path, - }) + const git = createGitClientForPath(path, onProgress); const cloneArgs = [ "--bare", @@ -62,11 +96,7 @@ export const fetchRepository = async ( } ) => { try { - const git = simpleGit({ - progress: onProgress, - }).cwd({ - path: path, - }) + const git = createGitClientForPath(path, onProgress); if (authHeader) { await git.addConfig("http.extraHeader", authHeader); @@ -108,9 +138,7 @@ export const fetchRepository = async ( * present in gitConfig. */ export const upsertGitConfig = async (path: string, gitConfig: Record, onProgress?: onProgressFn) => { - const git = simpleGit({ - progress: onProgress, - }).cwd(path); + const git = createGitClientForPath(path, onProgress); try { for (const [key, value] of Object.entries(gitConfig)) { @@ -130,9 +158,7 @@ export const upsertGitConfig = async (path: string, gitConfig: Record { - const git = simpleGit({ - progress: onProgress, - }).cwd(path); + const git = createGitClientForPath(path, onProgress); try { const configList = await git.listConfig(); @@ -156,9 +182,7 @@ export const unsetGitConfig = async (path: string, keys: string[], onProgress?: * Returns true if `path` is the _root_ of a git repository. */ export const isPathAValidGitRepoRoot = async (path: string, onProgress?: onProgressFn) => { - const git = simpleGit({ - progress: onProgress, - }).cwd(path); + const git = createGitClientForPath(path, onProgress); try { return git.checkIsRepo(CheckRepoActions.IS_REPO_ROOT); @@ -184,7 +208,7 @@ export const isUrlAValidGitRepo = async (url: string) => { } export const getOriginUrl = async (path: string) => { - const git = simpleGit().cwd(path); + const git = createGitClientForPath(path); try { const remotes = await git.getConfig('remote.origin.url', GitConfigScope.local); @@ -199,18 +223,13 @@ export const getOriginUrl = async (path: string) => { } export const getBranches = async (path: string) => { - const git = simpleGit(); - const branches = await git.cwd({ - path, - }).branch(); - + const git = createGitClientForPath(path); + const branches = await git.branch(); return branches.all; } export const getTags = async (path: string) => { - const git = simpleGit(); - const tags = await git.cwd({ - path, - }).tags(); + const git = createGitClientForPath(path); + const tags = await git.tags(); return tags.all; } \ No newline at end of file diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 0b9006b9..d8319272 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -6,16 +6,14 @@ import { hasEntitlement, loadConfig } from '@sourcebot/shared'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; import { Redis } from 'ioredis'; -import path from 'path'; import { ConnectionManager } from './connectionManager.js'; -import { DEFAULT_SETTINGS } from './constants.js'; -import { env } from "./env.js"; +import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js'; -import { PromClient } from './promClient.js'; -import { RepoManager } from './repoManager.js'; -import { AppContext } from "./types.js"; import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js"; +import { env } from "./env.js"; import { IndexSyncer } from "./indexSyncer.js"; +import { PromClient } from './promClient.js'; +import { RepoManager } from './repoManager.js'; const logger = createLogger('backend-entrypoint'); @@ -34,9 +32,8 @@ const getSettings = async (configPath?: string) => { } -const cacheDir = env.DATA_CACHE_DIR; -const reposPath = path.join(cacheDir, 'repos'); -const indexPath = path.join(cacheDir, 'index'); +const reposPath = REPOS_CACHE_DIR; +const indexPath = INDEX_CACHE_DIR; if (!existsSync(reposPath)) { await mkdir(reposPath, { recursive: true }); @@ -45,12 +42,6 @@ if (!existsSync(indexPath)) { await mkdir(indexPath, { recursive: true }); } -const context: AppContext = { - indexPath, - reposPath, - cachePath: cacheDir, -} - const prisma = new PrismaClient(); const redis = new Redis(env.REDIS_URL, { @@ -69,10 +60,10 @@ const promClient = new PromClient(); const settings = await getSettings(env.CONFIG_PATH); const connectionManager = new ConnectionManager(prisma, settings, redis); -const repoManager = new RepoManager(prisma, settings, redis, promClient, context); +const repoManager = new RepoManager(prisma, settings, redis, promClient); const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); const userPermissionSyncer = new UserPermissionSyncer(prisma, settings, redis); -const indexSyncer = new IndexSyncer(prisma, settings, redis, context); +const indexSyncer = new IndexSyncer(prisma, settings, redis); // await repoManager.validateIndexedReposHaveShards(); diff --git a/packages/backend/src/indexSyncer.ts b/packages/backend/src/indexSyncer.ts index 6bf2ac0a..7380618e 100644 --- a/packages/backend/src/indexSyncer.ts +++ b/packages/backend/src/indexSyncer.ts @@ -5,9 +5,10 @@ import { existsSync } from 'fs'; import { readdir, rm } from 'fs/promises'; import { Job, Queue, ReservedJob, Worker } from "groupmq"; import { Redis } from 'ioredis'; +import { INDEX_CACHE_DIR } from './constants.js'; import { env } from './env.js'; import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js'; -import { AppContext, repoMetadataSchema, RepoWithConnections, Settings } from "./types.js"; +import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js"; import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js'; import { indexGitRepository } from './zoekt.js'; @@ -33,16 +34,13 @@ export class IndexSyncer { private db: PrismaClient, private settings: Settings, redis: Redis, - private ctx: AppContext, ) { this.queue = new Queue({ redis, namespace: 'index-sync-queue', jobTimeoutMs: JOB_TIMEOUT_MS, maxAttempts: 3, - ...(env.SOURCEBOT_LOG_LEVEL === 'debug' ? { - logger, - }: {}), + logger: env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true', }); this.worker = new Worker({ @@ -50,8 +48,8 @@ export class IndexSyncer { maxStalledCount: 1, handler: this.runJob.bind(this), concurrency: this.settings.maxRepoIndexingJobConcurrency, - ...(env.SOURCEBOT_LOG_LEVEL === 'debug' ? { - logger, + ...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? { + logger: true, }: {}), }); @@ -62,6 +60,7 @@ export class IndexSyncer { } public async startScheduler() { + logger.debug('Starting scheduler'); this.interval = setInterval(async () => { await this.scheduleIndexJobs(); await this.scheduleCleanupJobs(); @@ -240,7 +239,7 @@ export class IndexSyncer { } private async indexRepository(repo: RepoWithConnections, logger: Logger) { - const { path: repoPath, isReadOnly } = getRepoPath(repo, this.ctx); + const { path: repoPath, isReadOnly } = getRepoPath(repo); const metadata = repoMetadataSchema.parse(repo.metadata); @@ -304,22 +303,22 @@ export class IndexSyncer { } logger.info(`Indexing ${repo.name} (id: ${repo.id})...`); - const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx)); + const { durationMs } = await measure(() => indexGitRepository(repo, this.settings)); const indexDuration_s = durationMs / 1000; logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); } private async cleanupRepository(repo: Repo, logger: Logger) { - const { path: repoPath, isReadOnly } = getRepoPath(repo, this.ctx); + const { path: repoPath, isReadOnly } = getRepoPath(repo); if (existsSync(repoPath) && !isReadOnly) { logger.info(`Deleting repo directory ${repoPath}`); await rm(repoPath, { recursive: true, force: true }); } const shardPrefix = getShardPrefix(repo.orgId, repo.id); - const files = (await readdir(this.ctx.indexPath)).filter(file => file.startsWith(shardPrefix)); + const files = (await readdir(INDEX_CACHE_DIR)).filter(file => file.startsWith(shardPrefix)); for (const file of files) { - const filePath = `${this.ctx.indexPath}/${file}`; + const filePath = `${INDEX_CACHE_DIR}/${file}`; logger.info(`Deleting shard file ${filePath}`); await rm(filePath, { force: true }); } diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts index 33f8bdad..570e2b15 100644 --- a/packages/backend/src/repoManager.ts +++ b/packages/backend/src/repoManager.ts @@ -4,10 +4,11 @@ import { createLogger } from "@sourcebot/logger"; import { Job, Queue, Worker } from 'bullmq'; import { existsSync, promises, readdirSync } from 'fs'; import { Redis } from 'ioredis'; +import { INDEX_CACHE_DIR } from "./constants.js"; import { env } from './env.js'; import { cloneRepository, fetchRepository, unsetGitConfig, upsertGitConfig } from "./git.js"; import { PromClient } from './promClient.js'; -import { AppContext, RepoWithConnections, Settings, repoMetadataSchema } from "./types.js"; +import { RepoWithConnections, Settings, repoMetadataSchema } from "./types.js"; import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, measure } from "./utils.js"; import { indexGitRepository } from "./zoekt.js"; @@ -36,7 +37,6 @@ export class RepoManager { private settings: Settings, redis: Redis, private promClient: PromClient, - private ctx: AppContext, ) { // Repo indexing this.indexQueue = new Queue(REPO_INDEXING_QUEUE, { @@ -162,7 +162,7 @@ export class RepoManager { } private async syncGitRepository(repo: RepoWithConnections, repoAlreadyInIndexingState: boolean) { - const { path: repoPath, isReadOnly } = getRepoPath(repo, this.ctx); + const { path: repoPath, isReadOnly } = getRepoPath(repo); const metadata = repoMetadataSchema.parse(repo.metadata); @@ -225,7 +225,7 @@ export class RepoManager { } logger.info(`Indexing ${repo.displayName}...`); - const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx)); + const { durationMs } = await measure(() => indexGitRepository(repo, this.settings)); const indexDuration_s = durationMs / 1000; logger.info(`Indexed ${repo.displayName} in ${indexDuration_s}s`); } @@ -422,7 +422,7 @@ export class RepoManager { }); // delete cloned repo - const { path: repoPath, isReadOnly } = getRepoPath(repo, this.ctx); + const { path: repoPath, isReadOnly } = getRepoPath(repo); if (existsSync(repoPath) && !isReadOnly) { logger.info(`Deleting repo directory ${repoPath}`); await promises.rm(repoPath, { recursive: true, force: true }); @@ -430,9 +430,9 @@ export class RepoManager { // delete shards const shardPrefix = getShardPrefix(repo.orgId, repo.id); - const files = readdirSync(this.ctx.indexPath).filter(file => file.startsWith(shardPrefix)); + const files = readdirSync(INDEX_CACHE_DIR).filter(file => file.startsWith(shardPrefix)); for (const file of files) { - const filePath = `${this.ctx.indexPath}/${file}`; + const filePath = `${INDEX_CACHE_DIR}/${file}`; logger.info(`Deleting shard file ${filePath}`); await promises.rm(filePath, { force: true }); } @@ -493,7 +493,7 @@ export class RepoManager { return; } - const files = readdirSync(this.ctx.indexPath); + const files = readdirSync(INDEX_CACHE_DIR); const reposToReindex: number[] = []; for (const repo of indexedRepos) { const shardPrefix = getShardPrefix(repo.orgId, repo.id); @@ -504,7 +504,7 @@ export class RepoManager { try { hasShards = files.some(file => file.startsWith(shardPrefix)); } catch (error) { - logger.error(`Failed to read index directory ${this.ctx.indexPath}: ${error}`); + logger.error(`Failed to read index directory ${INDEX_CACHE_DIR}: ${error}`); continue; } diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 2ea42d04..8e27867d 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -2,20 +2,6 @@ import { Connection, Repo, RepoToConnection } from "@sourcebot/db"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; import { z } from "zod"; -export type AppContext = { - /** - * Path to the repos cache directory. - */ - reposPath: string; - - /** - * Path to the index cache directory; - */ - indexPath: string; - - cachePath: string; -} - export type Settings = Required; // Structure of the `metadata` field in the `Repo` table. diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index f081ce1b..575e0cc9 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -1,11 +1,12 @@ import { Logger } from "winston"; -import { AppContext, RepoAuthCredentials, RepoWithConnections } from "./types.js"; +import { RepoAuthCredentials, RepoWithConnections } from "./types.js"; import path from 'path'; import { PrismaClient, Repo } from "@sourcebot/db"; import { getTokenFromConfig as getTokenFromConfigBase } from "@sourcebot/crypto"; import { BackendException, BackendError } from "@sourcebot/error"; import * as Sentry from "@sentry/node"; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; +import { REPOS_CACHE_DIR } from "./constants.js"; export const measure = async (cb: () => Promise) => { const start = Date.now(); @@ -69,7 +70,7 @@ export const arraysEqualShallow = (a?: readonly T[], b?: readonly T[]) => { // @note: this function is duplicated in `packages/web/src/features/fileTree/actions.ts`. // @todo: we should move this to a shared package. -export const getRepoPath = (repo: Repo, ctx: AppContext): { path: string, isReadOnly: boolean } => { +export const getRepoPath = (repo: Repo): { path: string, isReadOnly: boolean } => { // If we are dealing with a local repository, then use that as the path. // Mark as read-only since we aren't guaranteed to have write access to the local filesystem. const cloneUrl = new URL(repo.cloneUrl); @@ -81,7 +82,7 @@ export const getRepoPath = (repo: Repo, ctx: AppContext): { path: string, isRead } return { - path: path.join(ctx.reposPath, repo.id.toString()), + path: path.join(REPOS_CACHE_DIR, repo.id.toString()), isReadOnly: false, } } diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts index 076820c9..8b02e8ba 100644 --- a/packages/backend/src/zoekt.ts +++ b/packages/backend/src/zoekt.ts @@ -1,21 +1,21 @@ -import { exec } from "child_process"; -import { AppContext, repoMetadataSchema, Settings } from "./types.js"; import { Repo } from "@sourcebot/db"; -import { getRepoPath } from "./utils.js"; -import { getShardPrefix } from "./utils.js"; -import { getBranches, getTags } from "./git.js"; -import micromatch from "micromatch"; import { createLogger } from "@sourcebot/logger"; +import { exec } from "child_process"; +import micromatch from "micromatch"; +import { INDEX_CACHE_DIR } from "./constants.js"; +import { getBranches, getTags } from "./git.js"; import { captureEvent } from "./posthog.js"; +import { repoMetadataSchema, Settings } from "./types.js"; +import { getRepoPath, getShardPrefix } from "./utils.js"; const logger = createLogger('zoekt'); -export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => { +export const indexGitRepository = async (repo: Repo, settings: Settings) => { let revisions = [ 'HEAD' ]; - const { path: repoPath } = getRepoPath(repo, ctx); + const { path: repoPath } = getRepoPath(repo); const shardPrefix = getShardPrefix(repo.orgId, repo.id); const metadata = repoMetadataSchema.parse(repo.metadata); @@ -60,7 +60,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: Ap const command = [ 'zoekt-git-index', '-allow_missing_branches', - `-index ${ctx.indexPath}`, + `-index ${INDEX_CACHE_DIR}`, `-max_trigram_count ${settings.maxTrigramCount}`, `-file_limit ${settings.maxFileSize}`, `-branches "${revisions.join(',')}"`, From 5fe554e7da1dbfcd5b23cf40bb5fdd819d826c1d Mon Sep 17 00:00:00 2001 From: bkellam Date: Wed, 15 Oct 2025 11:23:34 -0700 Subject: [PATCH 06/24] renames + add abortSignal --- packages/backend/src/git.ts | 61 +- packages/backend/src/index.ts | 14 +- packages/backend/src/repoCompileUtils.ts | 4 +- .../{indexSyncer.ts => repoIndexManager.ts} | 75 ++- packages/backend/src/repoManager.ts | 566 ------------------ packages/backend/src/zoekt.ts | 4 +- 6 files changed, 118 insertions(+), 606 deletions(-) rename packages/backend/src/{indexSyncer.ts => repoIndexManager.ts} (87%) delete mode 100644 packages/backend/src/repoManager.ts diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts index e78b7db6..21f30462 100644 --- a/packages/backend/src/git.ts +++ b/packages/backend/src/git.ts @@ -10,7 +10,7 @@ type onProgressFn = (event: SimpleGitProgressEvent) => void; * Creates a simple-git client that has it's working directory * set to the given path. */ -const createGitClientForPath = (path: string, onProgress?: onProgressFn) => { +const createGitClientForPath = (path: string, onProgress?: onProgressFn, signal?: AbortSignal) => { if (!existsSync(path)) { throw new Error(`Path ${path} does not exist`); } @@ -19,6 +19,7 @@ const createGitClientForPath = (path: string, onProgress?: onProgressFn) => { const git = simpleGit({ progress: onProgress, + abort: signal, }) .env({ ...process.env, @@ -48,17 +49,19 @@ export const cloneRepository = async ( authHeader, path, onProgress, + signal, }: { cloneUrl: string, authHeader?: string, path: string, onProgress?: onProgressFn + signal?: AbortSignal } ) => { try { await mkdir(path, { recursive: true }); - const git = createGitClientForPath(path, onProgress); + const git = createGitClientForPath(path, onProgress, signal); const cloneArgs = [ "--bare", @@ -67,7 +70,11 @@ export const cloneRepository = async ( await git.clone(cloneUrl, path, cloneArgs); - await unsetGitConfig(path, ["remote.origin.url"]); + await unsetGitConfig({ + path, + keys: ["remote.origin.url"], + signal, + }); } catch (error: unknown) { const baseLog = `Failed to clone repository: ${path}`; @@ -88,15 +95,17 @@ export const fetchRepository = async ( authHeader, path, onProgress, + signal, }: { cloneUrl: string, authHeader?: string, path: string, - onProgress?: onProgressFn + onProgress?: onProgressFn, + signal?: AbortSignal } ) => { try { - const git = createGitClientForPath(path, onProgress); + const git = createGitClientForPath(path, onProgress, signal); if (authHeader) { await git.addConfig("http.extraHeader", authHeader); @@ -137,8 +146,19 @@ export const fetchRepository = async ( * that do not exist yet. It will _not_ remove any existing keys that are not * present in gitConfig. */ -export const upsertGitConfig = async (path: string, gitConfig: Record, onProgress?: onProgressFn) => { - const git = createGitClientForPath(path, onProgress); +export const upsertGitConfig = async ( + { + path, + gitConfig, + onProgress, + signal, + }: { + path: string, + gitConfig: Record, + onProgress?: onProgressFn, + signal?: AbortSignal + }) => { + const git = createGitClientForPath(path, onProgress, signal); try { for (const [key, value] of Object.entries(gitConfig)) { @@ -157,8 +177,19 @@ export const upsertGitConfig = async (path: string, gitConfig: Record { - const git = createGitClientForPath(path, onProgress); +export const unsetGitConfig = async ( + { + path, + keys, + onProgress, + signal, + }: { + path: string, + keys: string[], + onProgress?: onProgressFn, + signal?: AbortSignal + }) => { + const git = createGitClientForPath(path, onProgress, signal); try { const configList = await git.listConfig(); @@ -181,8 +212,16 @@ export const unsetGitConfig = async (path: string, keys: string[], onProgress?: /** * Returns true if `path` is the _root_ of a git repository. */ -export const isPathAValidGitRepoRoot = async (path: string, onProgress?: onProgressFn) => { - const git = createGitClientForPath(path, onProgress); +export const isPathAValidGitRepoRoot = async ({ + path, + onProgress, + signal, +}: { + path: string, + onProgress?: onProgressFn, + signal?: AbortSignal +}) => { + const git = createGitClientForPath(path, onProgress, signal); try { return git.checkIsRepo(CheckRepoActions.IS_REPO_ROOT); diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index d8319272..78a1ce6c 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -11,9 +11,8 @@ import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants. import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js'; import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js"; import { env } from "./env.js"; -import { IndexSyncer } from "./indexSyncer.js"; +import { RepoIndexManager } from "./repoIndexManager.js"; import { PromClient } from './promClient.js'; -import { RepoManager } from './repoManager.js'; const logger = createLogger('backend-entrypoint'); @@ -60,16 +59,12 @@ const promClient = new PromClient(); const settings = await getSettings(env.CONFIG_PATH); const connectionManager = new ConnectionManager(prisma, settings, redis); -const repoManager = new RepoManager(prisma, settings, redis, promClient); const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); const userPermissionSyncer = new UserPermissionSyncer(prisma, settings, redis); -const indexSyncer = new IndexSyncer(prisma, settings, redis); - -// await repoManager.validateIndexedReposHaveShards(); +const repoIndexManager = new RepoIndexManager(prisma, settings, redis); connectionManager.startScheduler(); -// repoManager.startScheduler(); -indexSyncer.startScheduler(); +repoIndexManager.startScheduler(); if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && !hasEntitlement('permission-syncing')) { logger.error('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.'); @@ -88,8 +83,7 @@ const cleanup = async (signal: string) => { try { await Promise.race([ Promise.all([ - indexSyncer.dispose(), - repoManager.dispose(), + repoIndexManager.dispose(), connectionManager.dispose(), repoPermissionSyncer.dispose(), userPermissionSyncer.dispose(), diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index 098f39c9..80d80ebe 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -497,7 +497,9 @@ export const compileGenericGitHostConfig_file = async ( }; await Promise.all(repoPaths.map(async (repoPath) => { - const isGitRepo = await isPathAValidGitRepoRoot(repoPath); + const isGitRepo = await isPathAValidGitRepoRoot({ + path: repoPath, + }); if (!isGitRepo) { logger.warn(`Skipping ${repoPath} - not a git repository.`); notFound.repos.push(repoPath); diff --git a/packages/backend/src/indexSyncer.ts b/packages/backend/src/repoIndexManager.ts similarity index 87% rename from packages/backend/src/indexSyncer.ts rename to packages/backend/src/repoIndexManager.ts index 7380618e..e296220d 100644 --- a/packages/backend/src/indexSyncer.ts +++ b/packages/backend/src/repoIndexManager.ts @@ -12,7 +12,7 @@ import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js"; import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js'; import { indexGitRepository } from './zoekt.js'; -const LOG_TAG = 'index-syncer'; +const LOG_TAG = 'repo-index-manager'; const logger = createLogger(LOG_TAG); const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); @@ -25,7 +25,18 @@ type JobPayload = { const JOB_TIMEOUT_MS = 1000 * 60 * 60 * 6; // 6 hour indexing timeout -export class IndexSyncer { +/** + * Manages the lifecycle of repository data on disk, including git working copies + * and search index shards. Handles both indexing operations (cloning/fetching repos + * and building search indexes) and cleanup operations (removing orphaned repos and + * their associated data). + * + * Uses a job queue system to process indexing and cleanup tasks asynchronously, + * with configurable concurrency limits and retry logic. Automatically schedules + * re-indexing of repos based on configured intervals and manages garbage collection + * of repos that are no longer connected to any source. + */ +export class RepoIndexManager { private interval?: NodeJS.Timeout; private queue: Queue; private worker: Worker; @@ -37,7 +48,7 @@ export class IndexSyncer { ) { this.queue = new Queue({ redis, - namespace: 'index-sync-queue', + namespace: 'repo-index-queue', jobTimeoutMs: JOB_TIMEOUT_MS, maxAttempts: 3, logger: env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true', @@ -210,6 +221,7 @@ export class IndexSyncer { const logger = createJobLogger(id); logger.info(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId}) (attempt ${job.attempts + 1} / ${job.maxAttempts})`); + const { repo, type: jobType } = await this.db.repoJob.update({ where: { id, @@ -231,14 +243,28 @@ export class IndexSyncer { } }); - if (jobType === RepoJobType.INDEX) { - await this.indexRepository(repo, logger); - } else if (jobType === RepoJobType.CLEANUP) { - await this.cleanupRepository(repo, logger); + const abortController = new AbortController(); + const signalHandler = () => { + logger.info(`Received shutdown signal, aborting...`); + abortController.abort(); // This cancels all operations + }; + + process.on('SIGTERM', signalHandler); + process.on('SIGINT', signalHandler); + + try { + if (jobType === RepoJobType.INDEX) { + await this.indexRepository(repo, logger, abortController.signal); + } else if (jobType === RepoJobType.CLEANUP) { + await this.cleanupRepository(repo, logger); + } + } finally { + process.off('SIGTERM', signalHandler); + process.off('SIGINT', signalHandler); } } - private async indexRepository(repo: RepoWithConnections, logger: Logger) { + private async indexRepository(repo: RepoWithConnections, logger: Logger, signal: AbortSignal) { const { path: repoPath, isReadOnly } = getRepoPath(repo); const metadata = repoMetadataSchema.parse(repo.metadata); @@ -250,9 +276,16 @@ export class IndexSyncer { // If the repo path exists but it is not a valid git repository root, this indicates // that the repository is in a bad state. To fix, we remove the directory and perform // a fresh clone. - if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot(repoPath)) && !isReadOnly) { - logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); - await rm(repoPath, { recursive: true, force: true }); + if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot( { path: repoPath } ))) { + const isValidGitRepo = await isPathAValidGitRepoRoot({ + path: repoPath, + signal, + }); + + if (!isValidGitRepo && !isReadOnly) { + logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); + await rm(repoPath, { recursive: true, force: true }); + } } if (existsSync(repoPath) && !isReadOnly) { @@ -262,7 +295,11 @@ export class IndexSyncer { // to unset this key since it is no longer needed, hence this line. // This will no-op if the key is already unset. // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 - await unsetGitConfig(repoPath, ["remote.origin.url"]); + await unsetGitConfig({ + path: repoPath, + keys: ["remote.origin.url"], + signal, + }); logger.info(`Fetching ${repo.name} (id: ${repo.id})...`); const { durationMs } = await measure(() => fetchRepository({ @@ -271,7 +308,8 @@ export class IndexSyncer { path: repoPath, onProgress: ({ method, stage, progress }) => { logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) - } + }, + signal, })); const fetchDuration_s = durationMs / 1000; @@ -287,7 +325,8 @@ export class IndexSyncer { path: repoPath, onProgress: ({ method, stage, progress }) => { logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) - } + }, + signal })); const cloneDuration_s = durationMs / 1000; @@ -299,11 +338,15 @@ export class IndexSyncer { // This ensures that the git config is always up to date for whatever we // have in the DB. if (metadata.gitConfig && !isReadOnly) { - await upsertGitConfig(repoPath, metadata.gitConfig); + await upsertGitConfig({ + path: repoPath, + gitConfig: metadata.gitConfig, + signal, + }); } logger.info(`Indexing ${repo.name} (id: ${repo.id})...`); - const { durationMs } = await measure(() => indexGitRepository(repo, this.settings)); + const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal)); const indexDuration_s = durationMs / 1000; logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); } diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts deleted file mode 100644 index 570e2b15..00000000 --- a/packages/backend/src/repoManager.ts +++ /dev/null @@ -1,566 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { PrismaClient, Repo, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; -import { Job, Queue, Worker } from 'bullmq'; -import { existsSync, promises, readdirSync } from 'fs'; -import { Redis } from 'ioredis'; -import { INDEX_CACHE_DIR } from "./constants.js"; -import { env } from './env.js'; -import { cloneRepository, fetchRepository, unsetGitConfig, upsertGitConfig } from "./git.js"; -import { PromClient } from './promClient.js'; -import { RepoWithConnections, Settings, repoMetadataSchema } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, measure } from "./utils.js"; -import { indexGitRepository } from "./zoekt.js"; - -const REPO_INDEXING_QUEUE = 'repoIndexingQueue'; -const REPO_GC_QUEUE = 'repoGarbageCollectionQueue'; - -type RepoIndexingPayload = { - repo: RepoWithConnections, -} - -type RepoGarbageCollectionPayload = { - repo: Repo, -} - -const logger = createLogger('repo-manager'); - -export class RepoManager { - private indexWorker: Worker; - private indexQueue: Queue; - private gcWorker: Worker; - private gcQueue: Queue; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - private promClient: PromClient, - ) { - // Repo indexing - this.indexQueue = new Queue(REPO_INDEXING_QUEUE, { - connection: redis, - }); - this.indexWorker = new Worker(REPO_INDEXING_QUEUE, this.runIndexJob.bind(this), { - connection: redis, - concurrency: this.settings.maxRepoIndexingJobConcurrency, - }); - this.indexWorker.on('completed', this.onIndexJobCompleted.bind(this)); - this.indexWorker.on('failed', this.onIndexJobFailed.bind(this)); - - // Garbage collection - this.gcQueue = new Queue(REPO_GC_QUEUE, { - connection: redis, - }); - this.gcWorker = new Worker(REPO_GC_QUEUE, this.runGarbageCollectionJob.bind(this), { - connection: redis, - concurrency: this.settings.maxRepoGarbageCollectionJobConcurrency, - }); - this.gcWorker.on('completed', this.onGarbageCollectionJobCompleted.bind(this)); - this.gcWorker.on('failed', this.onGarbageCollectionJobFailed.bind(this)); - } - - public startScheduler() { - logger.debug('Starting scheduler'); - this.interval = setInterval(async () => { - await this.fetchAndScheduleRepoIndexing(); - await this.fetchAndScheduleRepoGarbageCollection(); - await this.fetchAndScheduleRepoTimeouts(); - }, this.settings.reindexRepoPollingIntervalMs); - } - - /////////////////////////// - // Repo indexing - /////////////////////////// - - private async scheduleRepoIndexingBulk(repos: RepoWithConnections[]) { - await this.db.$transaction(async (tx) => { - await tx.repo.updateMany({ - where: { id: { in: repos.map(repo => repo.id) } }, - data: { repoIndexingStatus: RepoIndexingStatus.IN_INDEX_QUEUE } - }); - - const reposByOrg = repos.reduce>((acc, repo) => { - if (!acc[repo.orgId]) { - acc[repo.orgId] = []; - } - acc[repo.orgId].push(repo); - return acc; - }, {}); - - for (const orgId in reposByOrg) { - const orgRepos = reposByOrg[orgId]; - // Set priority based on number of repos (more repos = lower priority) - // This helps prevent large orgs from overwhelming the indexQueue - const priority = Math.min(Math.ceil(orgRepos.length / 10), 2097152); - - await this.indexQueue.addBulk(orgRepos.map(repo => ({ - name: 'repoIndexJob', - data: { repo }, - opts: { - priority: priority, - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - }, - }))); - - // Increment pending jobs counter for each repo added - orgRepos.forEach(repo => { - this.promClient.pendingRepoIndexingJobs.inc({ repo: repo.id.toString() }); - }); - - logger.info(`Added ${orgRepos.length} jobs to indexQueue for org ${orgId} with priority ${priority}`); - } - - - }).catch((err: unknown) => { - logger.error(`Failed to add jobs to indexQueue for repos ${repos.map(repo => repo.id).join(', ')}: ${err}`); - }); - } - - - private async fetchAndScheduleRepoIndexing() { - const thresholdDate = new Date(Date.now() - this.settings.reindexIntervalMs); - const repos = await this.db.repo.findMany({ - where: { - OR: [ - // "NEW" is really a misnomer here - it just means that the repo needs to be indexed - // immediately. In most cases, this will be because the repo was just created and - // is indeed "new". However, it could also be that a "retry" was requested on a failed - // index. So, we don't want to block on the indexedAt timestamp here. - { - repoIndexingStatus: RepoIndexingStatus.NEW, - }, - // When the repo has already been indexed, we only want to reindex if the reindexing - // interval has elapsed (or if the date isn't set for some reason). - { - AND: [ - { repoIndexingStatus: RepoIndexingStatus.INDEXED }, - { - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } }, - ] - } - ] - } - ] - }, - include: { - connections: { - include: { - connection: true - } - } - } - }); - - if (repos.length > 0) { - await this.scheduleRepoIndexingBulk(repos); - } - } - - private async syncGitRepository(repo: RepoWithConnections, repoAlreadyInIndexingState: boolean) { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - - const metadata = repoMetadataSchema.parse(repo.metadata); - - // If the repo was already in the indexing state, this job was likely killed and picked up again. As a result, - // to ensure the repo state is valid, we delete the repo if it exists so we get a fresh clone - if (repoAlreadyInIndexingState && existsSync(repoPath) && !isReadOnly) { - logger.info(`Deleting repo directory ${repoPath} during sync because it was already in the indexing state`); - await promises.rm(repoPath, { recursive: true, force: true }); - } - - const credentials = await getAuthCredentialsForRepo(repo, this.db); - const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; - const authHeader = credentials?.authHeader ?? undefined; - - if (existsSync(repoPath) && !isReadOnly) { - // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ - // write the clone URL (which could contain a auth token) to the - // `remote.origin.url` entry. For the upgrade scenario, we want - // to unset this key since it is no longer needed, hence this line. - // This will no-op if the key is already unset. - // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 - await unsetGitConfig(repoPath, ["remote.origin.url"]); - - logger.info(`Fetching ${repo.displayName}...`); - const { durationMs } = await measure(() => fetchRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`) - } - })); - const fetchDuration_s = durationMs / 1000; - - process.stdout.write('\n'); - logger.info(`Fetched ${repo.displayName} in ${fetchDuration_s}s`); - - } else if (!isReadOnly) { - logger.info(`Cloning ${repo.displayName}...`); - - const { durationMs } = await measure(() => cloneRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`) - } - })); - const cloneDuration_s = durationMs / 1000; - - process.stdout.write('\n'); - logger.info(`Cloned ${repo.displayName} in ${cloneDuration_s}s`); - } - - // Regardless of clone or fetch, always upsert the git config for the repo. - // This ensures that the git config is always up to date for whatever we - // have in the DB. - if (metadata.gitConfig && !isReadOnly) { - await upsertGitConfig(repoPath, metadata.gitConfig); - } - - logger.info(`Indexing ${repo.displayName}...`); - const { durationMs } = await measure(() => indexGitRepository(repo, this.settings)); - const indexDuration_s = durationMs / 1000; - logger.info(`Indexed ${repo.displayName} in ${indexDuration_s}s`); - } - - private async runIndexJob(job: Job) { - logger.info(`Running index job (id: ${job.id}) for repo ${job.data.repo.displayName}`); - const repo = job.data.repo as RepoWithConnections; - - // We have to use the existing repo object to get the repoIndexingStatus because the repo object - // inside the job is unchanged from when it was added to the queue. - const existingRepo = await this.db.repo.findUnique({ - where: { - id: repo.id, - }, - }); - if (!existingRepo) { - logger.error(`Repo ${repo.id} not found`); - const e = new Error(`Repo ${repo.id} not found`); - Sentry.captureException(e); - throw e; - } - const repoAlreadyInIndexingState = existingRepo.repoIndexingStatus === RepoIndexingStatus.INDEXING; - - - await this.db.repo.update({ - where: { - id: repo.id, - }, - data: { - repoIndexingStatus: RepoIndexingStatus.INDEXING, - } - }); - this.promClient.activeRepoIndexingJobs.inc(); - this.promClient.pendingRepoIndexingJobs.dec({ repo: repo.id.toString() }); - - let attempts = 0; - const maxAttempts = 3; - - while (attempts < maxAttempts) { - try { - await this.syncGitRepository(repo, repoAlreadyInIndexingState); - break; - } catch (error) { - Sentry.captureException(error); - - attempts++; - this.promClient.repoIndexingReattemptsTotal.inc(); - if (attempts === maxAttempts) { - logger.error(`Failed to sync repository ${repo.name} (id: ${repo.id}) after ${maxAttempts} attempts. Error: ${error}`); - throw error; - } - - const sleepDuration = (env.REPO_SYNC_RETRY_BASE_SLEEP_SECONDS * 1000) * Math.pow(2, attempts - 1); - logger.error(`Failed to sync repository ${repo.name} (id: ${repo.id}), attempt ${attempts}/${maxAttempts}. Sleeping for ${sleepDuration / 1000}s... Error: ${error}`); - await new Promise(resolve => setTimeout(resolve, sleepDuration)); - } - } - } - - private async onIndexJobCompleted(job: Job) { - logger.info(`Repo index job for repo ${job.data.repo.displayName} (id: ${job.data.repo.id}, jobId: ${job.id}) completed`); - this.promClient.activeRepoIndexingJobs.dec(); - this.promClient.repoIndexingSuccessTotal.inc(); - - await this.db.repo.update({ - where: { - id: job.data.repo.id, - }, - data: { - indexedAt: new Date(), - repoIndexingStatus: RepoIndexingStatus.INDEXED, - } - }); - } - - private async onIndexJobFailed(job: Job | undefined, err: unknown) { - logger.info(`Repo index job for repo ${job?.data.repo.displayName} (id: ${job?.data.repo.id}, jobId: ${job?.id}) failed with error: ${err}`); - Sentry.captureException(err, { - tags: { - repoId: job?.data.repo.id, - jobId: job?.id, - queue: REPO_INDEXING_QUEUE, - } - }); - - if (job) { - this.promClient.activeRepoIndexingJobs.dec(); - this.promClient.repoIndexingFailTotal.inc(); - - await this.db.repo.update({ - where: { - id: job.data.repo.id, - }, - data: { - repoIndexingStatus: RepoIndexingStatus.FAILED, - } - }) - } - } - - /////////////////////////// - // Repo garbage collection - /////////////////////////// - - private async scheduleRepoGarbageCollectionBulk(repos: Repo[]) { - await this.db.$transaction(async (tx) => { - await tx.repo.updateMany({ - where: { id: { in: repos.map(repo => repo.id) } }, - data: { repoIndexingStatus: RepoIndexingStatus.IN_GC_QUEUE } - }); - - await this.gcQueue.addBulk(repos.map(repo => ({ - name: 'repoGarbageCollectionJob', - data: { repo }, - opts: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - } - }))); - - logger.info(`Added ${repos.length} jobs to gcQueue`); - }); - } - - private async fetchAndScheduleRepoGarbageCollection() { - //////////////////////////////////// - // Get repos with no connections - //////////////////////////////////// - - - const thresholdDate = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs); - const reposWithNoConnections = await this.db.repo.findMany({ - where: { - repoIndexingStatus: { - in: [ - RepoIndexingStatus.INDEXED, // we don't include NEW repos here because they'll be picked up by the index queue (potential race condition) - RepoIndexingStatus.FAILED, - ] - }, - connections: { - none: {} - }, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } } - ] - }, - }); - if (reposWithNoConnections.length > 0) { - logger.info(`Garbage collecting ${reposWithNoConnections.length} repos with no connections: ${reposWithNoConnections.map(repo => repo.id).join(', ')}`); - } - - //////////////////////////////////// - // Get inactive org repos - //////////////////////////////////// - const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); - const inactiveOrgRepos = await this.db.repo.findMany({ - where: { - org: { - stripeSubscriptionStatus: StripeSubscriptionStatus.INACTIVE, - stripeLastUpdatedAt: { - lt: sevenDaysAgo - } - }, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } } - ] - } - }); - - if (inactiveOrgRepos.length > 0) { - logger.info(`Garbage collecting ${inactiveOrgRepos.length} inactive org repos: ${inactiveOrgRepos.map(repo => repo.id).join(', ')}`); - } - - const reposToDelete = [...reposWithNoConnections, ...inactiveOrgRepos]; - if (reposToDelete.length > 0) { - await this.scheduleRepoGarbageCollectionBulk(reposToDelete); - } - } - - private async runGarbageCollectionJob(job: Job) { - logger.info(`Running garbage collection job (id: ${job.id}) for repo ${job.data.repo.displayName} (id: ${job.data.repo.id})`); - this.promClient.activeRepoGarbageCollectionJobs.inc(); - - const repo = job.data.repo as Repo; - await this.db.repo.update({ - where: { - id: repo.id - }, - data: { - repoIndexingStatus: RepoIndexingStatus.GARBAGE_COLLECTING - } - }); - - // delete cloned repo - const { path: repoPath, isReadOnly } = getRepoPath(repo); - if (existsSync(repoPath) && !isReadOnly) { - logger.info(`Deleting repo directory ${repoPath}`); - await promises.rm(repoPath, { recursive: true, force: true }); - } - - // delete shards - const shardPrefix = getShardPrefix(repo.orgId, repo.id); - const files = readdirSync(INDEX_CACHE_DIR).filter(file => file.startsWith(shardPrefix)); - for (const file of files) { - const filePath = `${INDEX_CACHE_DIR}/${file}`; - logger.info(`Deleting shard file ${filePath}`); - await promises.rm(filePath, { force: true }); - } - } - - private async onGarbageCollectionJobCompleted(job: Job) { - logger.info(`Garbage collection job ${job.id} completed`); - this.promClient.activeRepoGarbageCollectionJobs.dec(); - this.promClient.repoGarbageCollectionSuccessTotal.inc(); - - await this.db.repo.delete({ - where: { - id: job.data.repo.id - } - }); - } - - private async onGarbageCollectionJobFailed(job: Job | undefined, err: unknown) { - logger.info(`Garbage collection job failed (id: ${job?.id ?? 'unknown'}) with error: ${err}`); - Sentry.captureException(err, { - tags: { - repoId: job?.data.repo.id, - jobId: job?.id, - queue: REPO_GC_QUEUE, - } - }); - - if (job) { - this.promClient.activeRepoGarbageCollectionJobs.dec(); - this.promClient.repoGarbageCollectionFailTotal.inc(); - - await this.db.repo.update({ - where: { - id: job.data.repo.id - }, - data: { - repoIndexingStatus: RepoIndexingStatus.GARBAGE_COLLECTION_FAILED - } - }); - } - } - - /////////////////////////// - // Repo index validation - /////////////////////////// - - public async validateIndexedReposHaveShards() { - logger.info('Validating indexed repos have shards...'); - - const indexedRepos = await this.db.repo.findMany({ - where: { - repoIndexingStatus: RepoIndexingStatus.INDEXED - } - }); - logger.info(`Found ${indexedRepos.length} repos in the DB marked as INDEXED`); - - if (indexedRepos.length === 0) { - return; - } - - const files = readdirSync(INDEX_CACHE_DIR); - const reposToReindex: number[] = []; - for (const repo of indexedRepos) { - const shardPrefix = getShardPrefix(repo.orgId, repo.id); - - // TODO: this doesn't take into account if a repo has multiple shards and only some of them are missing. To support that, this logic - // would need to know how many total shards are expected for this repo - let hasShards = false; - try { - hasShards = files.some(file => file.startsWith(shardPrefix)); - } catch (error) { - logger.error(`Failed to read index directory ${INDEX_CACHE_DIR}: ${error}`); - continue; - } - - if (!hasShards) { - logger.info(`Repo ${repo.displayName} (id: ${repo.id}) is marked as INDEXED but has no shards on disk. Marking for reindexing.`); - reposToReindex.push(repo.id); - } - } - - if (reposToReindex.length > 0) { - await this.db.repo.updateMany({ - where: { - id: { in: reposToReindex } - }, - data: { - repoIndexingStatus: RepoIndexingStatus.NEW - } - }); - logger.info(`Marked ${reposToReindex.length} repos for reindexing due to missing shards`); - } - - logger.info('Done validating indexed repos have shards'); - } - - private async fetchAndScheduleRepoTimeouts() { - const repos = await this.db.repo.findMany({ - where: { - repoIndexingStatus: RepoIndexingStatus.INDEXING, - updatedAt: { - lt: new Date(Date.now() - this.settings.repoIndexTimeoutMs) - } - } - }); - - if (repos.length > 0) { - logger.info(`Scheduling ${repos.length} repo timeouts`); - await this.scheduleRepoTimeoutsBulk(repos); - } - } - - private async scheduleRepoTimeoutsBulk(repos: Repo[]) { - await this.db.$transaction(async (tx) => { - await tx.repo.updateMany({ - where: { id: { in: repos.map(repo => repo.id) } }, - data: { repoIndexingStatus: RepoIndexingStatus.FAILED } - }); - }); - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - await this.indexWorker.close(); - await this.indexQueue.close(); - await this.gcQueue.close(); - await this.gcWorker.close(); - } -} \ No newline at end of file diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts index 8b02e8ba..ad75927a 100644 --- a/packages/backend/src/zoekt.ts +++ b/packages/backend/src/zoekt.ts @@ -10,7 +10,7 @@ import { getRepoPath, getShardPrefix } from "./utils.js"; const logger = createLogger('zoekt'); -export const indexGitRepository = async (repo: Repo, settings: Settings) => { +export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => { let revisions = [ 'HEAD' ]; @@ -71,7 +71,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings) => { ].join(' '); return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { - exec(command, (error, stdout, stderr) => { + exec(command, { signal }, (error, stdout, stderr) => { if (error) { reject(error); return; From c1467bcd827431beef61c0bf17e64fe2669d1564 Mon Sep 17 00:00:00 2001 From: bkellam Date: Wed, 15 Oct 2025 22:39:52 -0700 Subject: [PATCH 07/24] Various improvements and optimizations on the web side --- packages/web/src/actions.ts | 78 +++- .../components/pureTreePreviewPanel.tsx | 2 +- .../browse/hooks/useBrowseNavigation.ts | 34 +- .../[domain]/browse/hooks/useBrowsePath.ts | 3 +- .../src/app/[domain]/browse/hooks/utils.ts | 30 +- .../components/demoCards.tsx} | 6 +- .../components/landingPageChatBox.tsx} | 57 +-- .../[domain]/chat/components/newChatPanel.tsx | 72 ---- .../components/tutorialDialog.tsx} | 24 +- packages/web/src/app/[domain]/chat/layout.tsx | 5 + packages/web/src/app/[domain]/chat/page.tsx | 118 ++++-- .../[domain]/components/errorNavIndicator.tsx | 137 ------- .../[domain]/components/homepage/index.tsx | 102 ----- .../components/homepage/preciseSearch.tsx | 145 ------- .../homepage/repositoryCarousel.tsx | 105 ----- .../homepage/repositorySnapshot.tsx | 156 ------- .../index.tsx} | 121 ++++-- .../navigationMenu/progressIndicator.tsx | 118 ++++++ .../trialIndicator.tsx} | 2 +- .../app/[domain]/components/pathHeader.tsx | 2 +- .../components/progressNavIndicator.tsx | 73 ---- .../components/repositoryCarousel.tsx | 158 +++++++ .../toolbar.tsx => searchModeSelector.tsx} | 54 ++- .../components/warningNavIndicator.tsx | 79 ---- .../connections/[id]/components/repoList.tsx | 2 +- packages/web/src/app/[domain]/layout.tsx | 2 + packages/web/src/app/[domain]/page.tsx | 104 +---- .../web/src/app/[domain]/repos/columns.tsx | 2 +- .../search/components/searchLandingPage.tsx | 170 ++++++++ .../search/components/searchResultsPage.tsx | 372 +++++++++++++++++ .../searchResultsPanel/fileMatch.tsx | 2 +- packages/web/src/app/[domain]/search/page.tsx | 385 +----------------- .../web/src/app/[domain]/settings/layout.tsx | 4 + .../web/src/components/ui/navigation-menu.tsx | 2 +- .../components/exploreMenu/referenceList.tsx | 2 +- .../components/chatThread/tools/shared.tsx | 2 +- .../fileTree/components/pureFileTreePanel.tsx | 4 +- packages/web/src/lib/utils.ts | 13 + packages/web/tailwind.config.ts | 3 +- 39 files changed, 1196 insertions(+), 1554 deletions(-) rename packages/web/src/app/[domain]/{components/homepage/askSourcebotDemoCards.tsx => chat/components/demoCards.tsx} (98%) rename packages/web/src/app/[domain]/{components/homepage/agenticSearch.tsx => chat/components/landingPageChatBox.tsx} (59%) delete mode 100644 packages/web/src/app/[domain]/chat/components/newChatPanel.tsx rename packages/web/src/app/[domain]/{components/homepage/agenticSearchTutorialDialog.tsx => chat/components/tutorialDialog.tsx} (95%) delete mode 100644 packages/web/src/app/[domain]/components/errorNavIndicator.tsx delete mode 100644 packages/web/src/app/[domain]/components/homepage/index.tsx delete mode 100644 packages/web/src/app/[domain]/components/homepage/preciseSearch.tsx delete mode 100644 packages/web/src/app/[domain]/components/homepage/repositoryCarousel.tsx delete mode 100644 packages/web/src/app/[domain]/components/homepage/repositorySnapshot.tsx rename packages/web/src/app/[domain]/components/{navigationMenu.tsx => navigationMenu/index.tsx} (56%) create mode 100644 packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx rename packages/web/src/app/[domain]/components/{trialNavIndicator.tsx => navigationMenu/trialIndicator.tsx} (95%) delete mode 100644 packages/web/src/app/[domain]/components/progressNavIndicator.tsx create mode 100644 packages/web/src/app/[domain]/components/repositoryCarousel.tsx rename packages/web/src/app/[domain]/components/{homepage/toolbar.tsx => searchModeSelector.tsx} (83%) delete mode 100644 packages/web/src/app/[domain]/components/warningNavIndicator.tsx create mode 100644 packages/web/src/app/[domain]/search/components/searchLandingPage.tsx create mode 100644 packages/web/src/app/[domain]/search/components/searchResultsPage.tsx diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 5b73922c..3bfcee35 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -10,7 +10,7 @@ import { prisma } from "@/prisma"; import { render } from "@react-email/components"; import * as Sentry from '@sentry/nextjs'; import { decrypt, encrypt, generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto"; -import { ApiKey, ConnectionSyncStatus, Org, OrgRole, Prisma, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db"; +import { ApiKey, ConnectionSyncStatus, Org, OrgRole, Prisma, RepoIndexingStatus, RepoJobStatus, RepoJobType, StripeSubscriptionStatus } from "@sourcebot/db"; import { createLogger } from "@sourcebot/logger"; import { azuredevopsSchema } from "@sourcebot/schemas/v3/azuredevops.schema"; import { bitbucketSchema } from "@sourcebot/schemas/v3/bitbucket.schema"; @@ -638,22 +638,20 @@ export const getConnectionInfo = async (connectionId: number, domain: string) => } }))); -export const getRepos = async (filter: { status?: RepoIndexingStatus[], connectionId?: number } = {}) => sew(() => +export const getRepos = async ({ + where, + take, +}: { + where?: Prisma.RepoWhereInput, + take?: number +} = {}) => sew(() => withOptionalAuthV2(async ({ org, prisma }) => { const repos = await prisma.repo.findMany({ where: { orgId: org.id, - ...(filter.status ? { - repoIndexingStatus: { in: filter.status } - } : {}), - ...(filter.connectionId ? { - connections: { - some: { - connectionId: filter.connectionId - } - } - } : {}), - } + ...where, + }, + take, }); return repos.map((repo) => repositoryQuerySchema.parse({ @@ -669,6 +667,60 @@ export const getRepos = async (filter: { status?: RepoIndexingStatus[], connecti })) })); +/** + * Returns a set of aggregated stats about the repos in the org + */ +export const getReposStats = async () => sew(() => + withOptionalAuthV2(async ({ org, prisma }) => { + const [ + // Total number of repos. + numberOfRepos, + // Number of repos with their first time indexing jobs either + // pending or in progress. + numberOfReposWithFirstTimeIndexingJobsInProgress, + // Number of repos that have been indexed at least once. + numberOfReposWithIndex, + ] = await Promise.all([ + prisma.repo.count({ + where: { + orgId: org.id, + } + }), + prisma.repo.count({ + where: { + orgId: org.id, + jobs: { + some: { + type: RepoJobType.INDEX, + status: { + in: [ + RepoJobStatus.PENDING, + RepoJobStatus.IN_PROGRESS, + ] + } + }, + }, + indexedAt: null, + } + }), + prisma.repo.count({ + where: { + orgId: org.id, + NOT: { + indexedAt: null, + } + } + }) + ]); + + return { + numberOfRepos, + numberOfReposWithFirstTimeIndexingJobsInProgress, + numberOfReposWithIndex, + }; + }) +) + export const getRepoInfoByName = async (repoName: string) => sew(() => withOptionalAuthV2(async ({ org, prisma }) => { // @note: repo names are represented by their remote url diff --git a/packages/web/src/app/[domain]/browse/[...path]/components/pureTreePreviewPanel.tsx b/packages/web/src/app/[domain]/browse/[...path]/components/pureTreePreviewPanel.tsx index cdb8f3ca..83c9528e 100644 --- a/packages/web/src/app/[domain]/browse/[...path]/components/pureTreePreviewPanel.tsx +++ b/packages/web/src/app/[domain]/browse/[...path]/components/pureTreePreviewPanel.tsx @@ -3,7 +3,7 @@ import { useRef } from "react"; import { FileTreeItem } from "@/features/fileTree/actions"; import { FileTreeItemComponent } from "@/features/fileTree/components/fileTreeItemComponent"; -import { getBrowsePath } from "../../hooks/useBrowseNavigation"; +import { getBrowsePath } from "../../hooks/utils"; import { ScrollArea } from "@/components/ui/scroll-area"; import { useBrowseParams } from "../../hooks/useBrowseParams"; import { useDomain } from "@/hooks/useDomain"; diff --git a/packages/web/src/app/[domain]/browse/hooks/useBrowseNavigation.ts b/packages/web/src/app/[domain]/browse/hooks/useBrowseNavigation.ts index 0d79170e..c798a64c 100644 --- a/packages/web/src/app/[domain]/browse/hooks/useBrowseNavigation.ts +++ b/packages/web/src/app/[domain]/browse/hooks/useBrowseNavigation.ts @@ -3,7 +3,8 @@ import { useRouter } from "next/navigation"; import { useDomain } from "@/hooks/useDomain"; import { useCallback } from "react"; -import { BrowseState, SET_BROWSE_STATE_QUERY_PARAM } from "../browseStateProvider"; +import { BrowseState } from "../browseStateProvider"; +import { getBrowsePath } from "./utils"; export type BrowseHighlightRange = { start: { lineNumber: number; column: number; }; @@ -25,37 +26,6 @@ export interface GetBrowsePathProps { domain: string; } -export const getBrowsePath = ({ - repoName, - revisionName = 'HEAD', - path, - pathType, - highlightRange, - setBrowseState, - domain, -}: GetBrowsePathProps) => { - const params = new URLSearchParams(); - - if (highlightRange) { - const { start, end } = highlightRange; - - if ('column' in start && 'column' in end) { - params.set(HIGHLIGHT_RANGE_QUERY_PARAM, `${start.lineNumber}:${start.column},${end.lineNumber}:${end.column}`); - } else { - params.set(HIGHLIGHT_RANGE_QUERY_PARAM, `${start.lineNumber},${end.lineNumber}`); - } - } - - if (setBrowseState) { - params.set(SET_BROWSE_STATE_QUERY_PARAM, JSON.stringify(setBrowseState)); - } - - const encodedPath = encodeURIComponent(path); - const browsePath = `/${domain}/browse/${repoName}@${revisionName}/-/${pathType}/${encodedPath}${params.size > 0 ? `?${params.toString()}` : ''}`; - return browsePath; -} - - export const useBrowseNavigation = () => { const router = useRouter(); const domain = useDomain(); diff --git a/packages/web/src/app/[domain]/browse/hooks/useBrowsePath.ts b/packages/web/src/app/[domain]/browse/hooks/useBrowsePath.ts index fcf29be8..d612a75c 100644 --- a/packages/web/src/app/[domain]/browse/hooks/useBrowsePath.ts +++ b/packages/web/src/app/[domain]/browse/hooks/useBrowsePath.ts @@ -1,7 +1,8 @@ 'use client'; import { useMemo } from "react"; -import { getBrowsePath, GetBrowsePathProps } from "./useBrowseNavigation"; +import { GetBrowsePathProps } from "./useBrowseNavigation"; +import { getBrowsePath } from "./utils"; import { useDomain } from "@/hooks/useDomain"; export const useBrowsePath = ({ diff --git a/packages/web/src/app/[domain]/browse/hooks/utils.ts b/packages/web/src/app/[domain]/browse/hooks/utils.ts index ba3214fb..5e10b6d8 100644 --- a/packages/web/src/app/[domain]/browse/hooks/utils.ts +++ b/packages/web/src/app/[domain]/browse/hooks/utils.ts @@ -1,3 +1,5 @@ +import { SET_BROWSE_STATE_QUERY_PARAM } from "../browseStateProvider"; +import { GetBrowsePathProps, HIGHLIGHT_RANGE_QUERY_PARAM } from "./useBrowseNavigation"; export const getBrowseParamsFromPathParam = (pathParam: string) => { const sentinelIndex = pathParam.search(/\/-\/(tree|blob)/); @@ -7,7 +9,7 @@ export const getBrowseParamsFromPathParam = (pathParam: string) => { const repoAndRevisionPart = decodeURIComponent(pathParam.substring(0, sentinelIndex)); const lastAtIndex = repoAndRevisionPart.lastIndexOf('@'); - + const repoName = lastAtIndex === -1 ? repoAndRevisionPart : repoAndRevisionPart.substring(0, lastAtIndex); const revisionName = lastAtIndex === -1 ? undefined : repoAndRevisionPart.substring(lastAtIndex + 1); @@ -40,4 +42,28 @@ export const getBrowseParamsFromPathParam = (pathParam: string) => { path, pathType, } -} \ No newline at end of file +}; + +export const getBrowsePath = ({ + repoName, revisionName = 'HEAD', path, pathType, highlightRange, setBrowseState, domain, +}: GetBrowsePathProps) => { + const params = new URLSearchParams(); + + if (highlightRange) { + const { start, end } = highlightRange; + + if ('column' in start && 'column' in end) { + params.set(HIGHLIGHT_RANGE_QUERY_PARAM, `${start.lineNumber}:${start.column},${end.lineNumber}:${end.column}`); + } else { + params.set(HIGHLIGHT_RANGE_QUERY_PARAM, `${start.lineNumber},${end.lineNumber}`); + } + } + + if (setBrowseState) { + params.set(SET_BROWSE_STATE_QUERY_PARAM, JSON.stringify(setBrowseState)); + } + + const encodedPath = encodeURIComponent(path); + const browsePath = `/${domain}/browse/${repoName}@${revisionName}/-/${pathType}/${encodedPath}${params.size > 0 ? `?${params.toString()}` : ''}`; + return browsePath; +}; diff --git a/packages/web/src/app/[domain]/components/homepage/askSourcebotDemoCards.tsx b/packages/web/src/app/[domain]/chat/components/demoCards.tsx similarity index 98% rename from packages/web/src/app/[domain]/components/homepage/askSourcebotDemoCards.tsx rename to packages/web/src/app/[domain]/chat/components/demoCards.tsx index 31037607..016d9605 100644 --- a/packages/web/src/app/[domain]/components/homepage/askSourcebotDemoCards.tsx +++ b/packages/web/src/app/[domain]/chat/components/demoCards.tsx @@ -11,13 +11,13 @@ import { cn, getCodeHostIcon } from "@/lib/utils"; import useCaptureEvent from "@/hooks/useCaptureEvent"; import { SearchScopeInfoCard } from "@/features/chat/components/chatBox/searchScopeInfoCard"; -interface AskSourcebotDemoCardsProps { +interface DemoCards { demoExamples: DemoExamples; } -export const AskSourcebotDemoCards = ({ +export const DemoCards = ({ demoExamples, -}: AskSourcebotDemoCardsProps) => { +}: DemoCards) => { const captureEvent = useCaptureEvent(); const [selectedFilterSearchScope, setSelectedFilterSearchScope] = useState(null); diff --git a/packages/web/src/app/[domain]/components/homepage/agenticSearch.tsx b/packages/web/src/app/[domain]/chat/components/landingPageChatBox.tsx similarity index 59% rename from packages/web/src/app/[domain]/components/homepage/agenticSearch.tsx rename to packages/web/src/app/[domain]/chat/components/landingPageChatBox.tsx index 327cd297..b2080df3 100644 --- a/packages/web/src/app/[domain]/components/homepage/agenticSearch.tsx +++ b/packages/web/src/app/[domain]/chat/components/landingPageChatBox.tsx @@ -6,47 +6,24 @@ import { ChatBoxToolbar } from "@/features/chat/components/chatBox/chatBoxToolba import { LanguageModelInfo, SearchScope } from "@/features/chat/types"; import { useCreateNewChatThread } from "@/features/chat/useCreateNewChatThread"; import { RepositoryQuery, SearchContextQuery } from "@/lib/types"; -import { useCallback, useState } from "react"; -import { SearchModeSelector, SearchModeSelectorProps } from "./toolbar"; +import { useState } from "react"; import { useLocalStorage } from "usehooks-ts"; -import { DemoExamples } from "@/types"; -import { AskSourcebotDemoCards } from "./askSourcebotDemoCards"; -import { AgenticSearchTutorialDialog } from "./agenticSearchTutorialDialog"; -import { setAgenticSearchTutorialDismissedCookie } from "@/actions"; -import { RepositorySnapshot } from "./repositorySnapshot"; +import { SearchModeSelector } from "../../components/searchModeSelector"; -interface AgenticSearchProps { - searchModeSelectorProps: SearchModeSelectorProps; +interface LandingPageChatBox { languageModels: LanguageModelInfo[]; repos: RepositoryQuery[]; searchContexts: SearchContextQuery[]; - chatHistory: { - id: string; - createdAt: Date; - name: string | null; - }[]; - demoExamples: DemoExamples | undefined; - isTutorialDismissed: boolean; } -export const AgenticSearch = ({ - searchModeSelectorProps, +export const LandingPageChatBox = ({ languageModels, repos, searchContexts, - demoExamples, - isTutorialDismissed, -}: AgenticSearchProps) => { +}: LandingPageChatBox) => { const { createNewChatThread, isLoading } = useCreateNewChatThread(); const [selectedSearchScopes, setSelectedSearchScopes] = useLocalStorage("selectedSearchScopes", [], { initializeWithValue: false }); const [isContextSelectorOpen, setIsContextSelectorOpen] = useState(false); - - const [isTutorialOpen, setIsTutorialOpen] = useState(!isTutorialDismissed); - const onTutorialDismissed = useCallback(() => { - setIsTutorialOpen(false); - setAgenticSearchTutorialDismissedCookie(true); - }, []); - return (
@@ -74,34 +51,12 @@ export const AgenticSearch = ({ onContextSelectorOpenChanged={setIsContextSelectorOpen} />
- -
- -
- -
- -
- - {demoExamples && ( - - )} - - {isTutorialOpen && ( - - )} ) } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/chat/components/newChatPanel.tsx b/packages/web/src/app/[domain]/chat/components/newChatPanel.tsx deleted file mode 100644 index 91ae3f96..00000000 --- a/packages/web/src/app/[domain]/chat/components/newChatPanel.tsx +++ /dev/null @@ -1,72 +0,0 @@ -'use client'; - -import { ResizablePanel } from "@/components/ui/resizable"; -import { ChatBox } from "@/features/chat/components/chatBox"; -import { ChatBoxToolbar } from "@/features/chat/components/chatBox/chatBoxToolbar"; -import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; -import { useCreateNewChatThread } from "@/features/chat/useCreateNewChatThread"; -import { LanguageModelInfo, SearchScope } from "@/features/chat/types"; -import { RepositoryQuery, SearchContextQuery } from "@/lib/types"; -import { useCallback, useState } from "react"; -import { Descendant } from "slate"; -import { useLocalStorage } from "usehooks-ts"; - -interface NewChatPanelProps { - languageModels: LanguageModelInfo[]; - repos: RepositoryQuery[]; - searchContexts: SearchContextQuery[]; - order: number; -} - -export const NewChatPanel = ({ - languageModels, - repos, - searchContexts, - order, -}: NewChatPanelProps) => { - const [selectedSearchScopes, setSelectedSearchScopes] = useLocalStorage("selectedSearchScopes", [], { initializeWithValue: false }); - const { createNewChatThread, isLoading } = useCreateNewChatThread(); - const [isContextSelectorOpen, setIsContextSelectorOpen] = useState(false); - - const onSubmit = useCallback((children: Descendant[]) => { - createNewChatThread(children, selectedSearchScopes); - }, [createNewChatThread, selectedSearchScopes]); - - - return ( - -
-

What can I help you understand?

-
- - -
- -
- -
-
- - ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/homepage/agenticSearchTutorialDialog.tsx b/packages/web/src/app/[domain]/chat/components/tutorialDialog.tsx similarity index 95% rename from packages/web/src/app/[domain]/components/homepage/agenticSearchTutorialDialog.tsx rename to packages/web/src/app/[domain]/chat/components/tutorialDialog.tsx index a44d6735..c5ddd85d 100644 --- a/packages/web/src/app/[domain]/components/homepage/agenticSearchTutorialDialog.tsx +++ b/packages/web/src/app/[domain]/chat/components/tutorialDialog.tsx @@ -1,7 +1,8 @@ "use client" +import { setAgenticSearchTutorialDismissedCookie } from "@/actions" import { Button } from "@/components/ui/button" -import { Dialog, DialogContent } from "@/components/ui/dialog" +import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog" import { ModelProviderLogo } from "@/features/chat/components/chatBox/modelProviderLogo" import { cn } from "@/lib/utils" import mentionsDemo from "@/public/ask_sb_tutorial_at_mentions.png" @@ -27,11 +28,9 @@ import { } from "lucide-react" import Image from "next/image" import Link from "next/link" -import { useState } from "react" +import { useCallback, useState } from "react" + -interface AgenticSearchTutorialDialogProps { - onClose: () => void -} // Star button component that fetches GitHub star count @@ -249,7 +248,17 @@ const tutorialSteps = [ }, ] -export const AgenticSearchTutorialDialog = ({ onClose }: AgenticSearchTutorialDialogProps) => { +interface TutorialDialogProps { + isOpen: boolean; +} + +export const TutorialDialog = ({ isOpen: _isOpen }: TutorialDialogProps) => { + const [isOpen, setIsOpen] = useState(_isOpen); + const onClose = useCallback(() => { + setIsOpen(false); + setAgenticSearchTutorialDismissedCookie(true); + }, []); + const [currentStep, setCurrentStep] = useState(0) const nextStep = () => { @@ -269,11 +278,12 @@ export const AgenticSearchTutorialDialog = ({ onClose }: AgenticSearchTutorialDi const currentStepData = tutorialSteps[currentStep]; return ( - + + Ask Sourcebot tutorial
{/* Left Column (Text Content & Navigation) */}
diff --git a/packages/web/src/app/[domain]/chat/layout.tsx b/packages/web/src/app/[domain]/chat/layout.tsx index e82ea91f..2968c748 100644 --- a/packages/web/src/app/[domain]/chat/layout.tsx +++ b/packages/web/src/app/[domain]/chat/layout.tsx @@ -1,10 +1,14 @@ +import { AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME } from '@/lib/constants'; import { NavigationGuardProvider } from 'next-navigation-guard'; +import { cookies } from 'next/headers'; +import { TutorialDialog } from './components/tutorialDialog'; interface LayoutProps { children: React.ReactNode; } export default async function Layout({ children }: LayoutProps) { + const isTutorialDismissed = (await cookies()).get(AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME)?.value === "true"; return ( // @note: we use a navigation guard here since we don't support resuming streams yet. @@ -13,6 +17,7 @@ export default async function Layout({ children }: LayoutProps) {
{children}
+ ) } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/chat/page.tsx b/packages/web/src/app/[domain]/chat/page.tsx index 8bdddf9e..5b7afef2 100644 --- a/packages/web/src/app/[domain]/chat/page.tsx +++ b/packages/web/src/app/[domain]/chat/page.tsx @@ -1,13 +1,17 @@ -import { getRepos, getSearchContexts } from "@/actions"; -import { getUserChatHistory, getConfiguredLanguageModelsInfo } from "@/features/chat/actions"; +import { getRepos, getReposStats, getSearchContexts } from "@/actions"; +import { SourcebotLogo } from "@/app/components/sourcebotLogo"; +import { getConfiguredLanguageModelsInfo } from "@/features/chat/actions"; +import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; import { ServiceErrorException } from "@/lib/serviceError"; -import { isServiceError } from "@/lib/utils"; -import { NewChatPanel } from "./components/newChatPanel"; -import { TopBar } from "../components/topBar"; -import { ResizablePanelGroup } from "@/components/ui/resizable"; -import { ChatSidePanel } from "./components/chatSidePanel"; -import { auth } from "@/auth"; -import { AnimatedResizableHandle } from "@/components/ui/animatedResizableHandle"; +import { isServiceError, measure } from "@/lib/utils"; +import { LandingPageChatBox } from "./components/landingPageChatBox"; +import { RepositoryCarousel } from "../components/repositoryCarousel"; +import { NavigationMenu } from "../components/navigationMenu"; +import { Separator } from "@/components/ui/separator"; +import { DemoCards } from "./components/demoCards"; +import { env } from "@/env.mjs"; +import { loadJsonFile } from "@sourcebot/shared"; +import { DemoExamples, demoExamplesSchema } from "@/types"; interface PageProps { params: Promise<{ @@ -18,47 +22,85 @@ interface PageProps { export default async function Page(props: PageProps) { const params = await props.params; const languageModels = await getConfiguredLanguageModelsInfo(); - const repos = await getRepos(); const searchContexts = await getSearchContexts(params.domain); - const session = await auth(); - const chatHistory = session ? await getUserChatHistory(params.domain) : []; + const allRepos = await getRepos(); - if (isServiceError(chatHistory)) { - throw new ServiceErrorException(chatHistory); - } + const carouselRepos = await getRepos({ + where: { + indexedAt: { + not: null, + }, + }, + take: 10, + }); + + const repoStats = await getReposStats(); - if (isServiceError(repos)) { - throw new ServiceErrorException(repos); + if (isServiceError(allRepos)) { + throw new ServiceErrorException(allRepos); } if (isServiceError(searchContexts)) { throw new ServiceErrorException(searchContexts); } - const indexedRepos = repos.filter((repo) => repo.indexedAt !== undefined); + if (isServiceError(carouselRepos)) { + throw new ServiceErrorException(carouselRepos); + } + + if (isServiceError(repoStats)) { + throw new ServiceErrorException(repoStats); + } + + const demoExamples = env.SOURCEBOT_DEMO_EXAMPLES_PATH ? await (async () => { + try { + return (await measure(() => loadJsonFile(env.SOURCEBOT_DEMO_EXAMPLES_PATH!, demoExamplesSchema), 'loadExamplesJsonFile')).data; + } catch (error) { + console.error('Failed to load demo examples:', error); + return undefined; + } + })() : undefined; return ( - <> - + - - - - - - + +
+
+ +
+ + + + +
+ +
+ + {demoExamples && ( + <> +
+ +
+ + + + )} + +
+
) } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/errorNavIndicator.tsx b/packages/web/src/app/[domain]/components/errorNavIndicator.tsx deleted file mode 100644 index 4f7810aa..00000000 --- a/packages/web/src/app/[domain]/components/errorNavIndicator.tsx +++ /dev/null @@ -1,137 +0,0 @@ -"use client"; - -import Link from "next/link"; -import { HoverCard, HoverCardTrigger, HoverCardContent } from "@/components/ui/hover-card"; -import { CircleXIcon } from "lucide-react"; -import { useDomain } from "@/hooks/useDomain"; -import { unwrapServiceError } from "@/lib/utils"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { env } from "@/env.mjs"; -import { useQuery } from "@tanstack/react-query"; -import { ConnectionSyncStatus, RepoIndexingStatus } from "@sourcebot/db"; -import { getConnections } from "@/actions"; -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; -import { getRepos } from "@/app/api/(client)/client"; - -export const ErrorNavIndicator = () => { - const domain = useDomain(); - const captureEvent = useCaptureEvent(); - - const { data: repos, isPending: isPendingRepos, isError: isErrorRepos } = useQuery({ - queryKey: ['repos', domain], - queryFn: () => unwrapServiceError(getRepos()), - select: (data) => data.filter(repo => repo.repoIndexingStatus === RepoIndexingStatus.FAILED), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - const { data: connections, isPending: isPendingConnections, isError: isErrorConnections } = useQuery({ - queryKey: ['connections', domain], - queryFn: () => unwrapServiceError(getConnections(domain)), - select: (data) => data.filter(connection => connection.syncStatus === ConnectionSyncStatus.FAILED), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - if (isPendingRepos || isErrorRepos || isPendingConnections || isErrorConnections) { - return null; - } - - if (repos.length === 0 && connections.length === 0) { - return null; - } - - return ( - - captureEvent('wa_error_nav_hover', {})}> - captureEvent('wa_error_nav_pressed', {})}> -
- - {repos.length + connections.length > 0 && ( - {repos.length + connections.length} - )} -
- - - -
- {connections.length > 0 && ( -
-
-
-

Connection Sync Issues

-
-

- The following connections have failed to sync: -

-
- - {connections - .slice(0, 10) - .map(connection => ( - captureEvent('wa_error_nav_job_pressed', {})}> -
- - - {connection.name} - - - {connection.name} - - -
- - ))} - - {connections.length > 10 && ( -
- And {connections.length - 10} more... -
- )} -
-
- )} - - {repos.length > 0 && ( -
-
-
-

Repository Indexing Issues

-
-

- The following repositories failed to index: -

-
- - {repos - .slice(0, 10) - .map(repo => ( -
- - - {repo.repoName} - - - {repo.repoName} - - -
- ))} - - {repos.length > 10 && ( -
- And {repos.length - 10} more... -
- )} -
-
- )} -
- - - ); -}; diff --git a/packages/web/src/app/[domain]/components/homepage/index.tsx b/packages/web/src/app/[domain]/components/homepage/index.tsx deleted file mode 100644 index fa230288..00000000 --- a/packages/web/src/app/[domain]/components/homepage/index.tsx +++ /dev/null @@ -1,102 +0,0 @@ -'use client'; - -import { SourcebotLogo } from "@/app/components/sourcebotLogo"; -import { LanguageModelInfo } from "@/features/chat/types"; -import { RepositoryQuery, SearchContextQuery } from "@/lib/types"; -import { useHotkeys } from "react-hotkeys-hook"; -import { AgenticSearch } from "./agenticSearch"; -import { PreciseSearch } from "./preciseSearch"; -import { SearchMode } from "./toolbar"; -import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; -import { setSearchModeCookie } from "@/actions"; -import { useCallback, useState } from "react"; -import { DemoExamples } from "@/types"; - -interface HomepageProps { - initialRepos: RepositoryQuery[]; - searchContexts: SearchContextQuery[]; - languageModels: LanguageModelInfo[]; - chatHistory: { - id: string; - createdAt: Date; - name: string | null; - }[]; - initialSearchMode: SearchMode; - demoExamples: DemoExamples | undefined; - isAgenticSearchTutorialDismissed: boolean; -} - - -export const Homepage = ({ - initialRepos, - searchContexts, - languageModels, - chatHistory, - initialSearchMode, - demoExamples, - isAgenticSearchTutorialDismissed, -}: HomepageProps) => { - const [searchMode, setSearchMode] = useState(initialSearchMode); - const isAgenticSearchEnabled = languageModels.length > 0; - - const onSearchModeChanged = useCallback(async (newMode: SearchMode) => { - setSearchMode(newMode); - await setSearchModeCookie(newMode); - }, [setSearchMode]); - - useHotkeys("mod+i", (e) => { - e.preventDefault(); - onSearchModeChanged("agentic"); - }, { - enableOnFormTags: true, - enableOnContentEditable: true, - description: "Switch to agentic search", - }); - - useHotkeys("mod+p", (e) => { - e.preventDefault(); - onSearchModeChanged("precise"); - }, { - enableOnFormTags: true, - enableOnContentEditable: true, - description: "Switch to precise search", - }); - - return ( -
-
- -
- - {searchMode === "precise" ? ( - - ) : ( - - - - )} -
- ) -} - diff --git a/packages/web/src/app/[domain]/components/homepage/preciseSearch.tsx b/packages/web/src/app/[domain]/components/homepage/preciseSearch.tsx deleted file mode 100644 index d5608940..00000000 --- a/packages/web/src/app/[domain]/components/homepage/preciseSearch.tsx +++ /dev/null @@ -1,145 +0,0 @@ -'use client'; - -import { Separator } from "@/components/ui/separator"; -import { SyntaxReferenceGuideHint } from "../syntaxReferenceGuideHint"; -import { RepositorySnapshot } from "./repositorySnapshot"; -import { RepositoryQuery } from "@/lib/types"; -import { useDomain } from "@/hooks/useDomain"; -import Link from "next/link"; -import { SearchBar } from "../searchBar/searchBar"; -import { SearchModeSelector, SearchModeSelectorProps } from "./toolbar"; - -interface PreciseSearchProps { - initialRepos: RepositoryQuery[]; - searchModeSelectorProps: SearchModeSelectorProps; -} - -export const PreciseSearch = ({ - initialRepos, - searchModeSelectorProps, -}: PreciseSearchProps) => { - const domain = useDomain(); - - return ( - <> -
- - -
- -
-
-
- -
-
- - How to search -
- - - test todo (both test and todo) - - - test or todo (either test or todo) - - - {`"exit boot"`} (exact match) - - - TODO case:yes (case sensitive) - - - - - file:README setup (by filename) - - - repo:torvalds/linux test (by repo) - - - lang:typescript (by language) - - - rev:HEAD (by branch or tag) - - - - - file:{`\\.py$`} {`(files that end in ".py")`} - - - sym:main {`(symbols named "main")`} - - - todo -lang:c (negate filter) - - - content:README (search content only) - - -
- -
- - ) -} - -const HowToSection = ({ title, children }: { title: string, children: React.ReactNode }) => { - return ( -
- {title} - {children} -
- ) - -} - -const Highlight = ({ children }: { children: React.ReactNode }) => { - return ( - - {children} - - ) -} - -const QueryExample = ({ children }: { children: React.ReactNode }) => { - return ( - - {children} - - ) -} - -const QueryExplanation = ({ children }: { children: React.ReactNode }) => { - return ( - - {children} - - ) -} - -const Query = ({ query, domain, children }: { query: string, domain: string, children: React.ReactNode }) => { - return ( - - {children} - - ) -} diff --git a/packages/web/src/app/[domain]/components/homepage/repositoryCarousel.tsx b/packages/web/src/app/[domain]/components/homepage/repositoryCarousel.tsx deleted file mode 100644 index 0a90e1a7..00000000 --- a/packages/web/src/app/[domain]/components/homepage/repositoryCarousel.tsx +++ /dev/null @@ -1,105 +0,0 @@ -'use client'; - -import { - Carousel, - CarouselContent, - CarouselItem, -} from "@/components/ui/carousel"; -import Autoscroll from "embla-carousel-auto-scroll"; -import { getCodeHostInfoForRepo } from "@/lib/utils"; -import Image from "next/image"; -import { FileIcon } from "@radix-ui/react-icons"; -import clsx from "clsx"; -import { RepositoryQuery } from "@/lib/types"; -import { getBrowsePath } from "../../browse/hooks/useBrowseNavigation"; -import Link from "next/link"; -import { useDomain } from "@/hooks/useDomain"; - -interface RepositoryCarouselProps { - repos: RepositoryQuery[]; -} - -export const RepositoryCarousel = ({ - repos, -}: RepositoryCarouselProps) => { - return ( - - - {repos.map((repo, index) => ( - - - - ))} - - - ) -}; - -interface RepositoryBadgeProps { - repo: RepositoryQuery; -} - -const RepositoryBadge = ({ - repo -}: RepositoryBadgeProps) => { - const domain = useDomain(); - const { repoIcon, displayName } = (() => { - const info = getCodeHostInfoForRepo({ - codeHostType: repo.codeHostType, - name: repo.repoName, - displayName: repo.repoDisplayName, - webUrl: repo.webUrl, - }); - - if (info) { - return { - repoIcon: {info.codeHostName}, - displayName: info.displayName, - } - } - - return { - repoIcon: , - displayName: repo.repoName, - } - })(); - - return ( - - {repoIcon} - - {displayName} - - - ) -} diff --git a/packages/web/src/app/[domain]/components/homepage/repositorySnapshot.tsx b/packages/web/src/app/[domain]/components/homepage/repositorySnapshot.tsx deleted file mode 100644 index d6845fa2..00000000 --- a/packages/web/src/app/[domain]/components/homepage/repositorySnapshot.tsx +++ /dev/null @@ -1,156 +0,0 @@ -"use client"; - -import Link from "next/link"; -import { RepositoryCarousel } from "./repositoryCarousel"; -import { useDomain } from "@/hooks/useDomain"; -import { useQuery } from "@tanstack/react-query"; -import { unwrapServiceError } from "@/lib/utils"; -import { getRepos } from "@/app/api/(client)/client"; -import { env } from "@/env.mjs"; -import { Skeleton } from "@/components/ui/skeleton"; -import { - Carousel, - CarouselContent, - CarouselItem, -} from "@/components/ui/carousel"; -import { RepoIndexingStatus } from "@sourcebot/db"; -import { SymbolIcon } from "@radix-ui/react-icons"; -import { RepositoryQuery } from "@/lib/types"; -import { captureEvent } from "@/hooks/useCaptureEvent"; - -interface RepositorySnapshotProps { - repos: RepositoryQuery[]; -} - -const MAX_REPOS_TO_DISPLAY_IN_CAROUSEL = 15; - -export function RepositorySnapshot({ - repos: initialRepos, -}: RepositorySnapshotProps) { - const domain = useDomain(); - - const { data: repos, isPending, isError } = useQuery({ - queryKey: ['repos', domain], - queryFn: () => unwrapServiceError(getRepos()), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - placeholderData: initialRepos, - }); - - if (isPending || isError || !repos) { - return ( -
- -
- ) - } - - // Use `indexedAt` to determine if a repo has __ever__ been indexed. - // The repo indexing status only tells us the repo's current indexing status. - const indexedRepos = repos.filter((repo) => repo.indexedAt !== undefined); - - // If there are no indexed repos... - if (indexedRepos.length === 0) { - - // ... show a loading state if repos are being indexed now - if (repos.some((repo) => repo.repoIndexingStatus === RepoIndexingStatus.INDEXING || repo.repoIndexingStatus === RepoIndexingStatus.IN_INDEX_QUEUE)) { - return ( -
- - indexing in progress... -
- ) - - // ... otherwise, show the empty state. - } else { - return ( - - ) - } - } - - return ( -
- - {`${indexedRepos.length} `} - - {indexedRepos.length > 1 ? 'repositories' : 'repository'} - - {` indexed`} - - - {process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === "demo" && ( -

- Interested in using Sourcebot on your code? Check out our{' '} - captureEvent('wa_demo_docs_link_pressed', {})} - > - docs - -

- )} -
- ) -} - -function EmptyRepoState() { - return ( -
- No repositories found - -
-
- - <> - Create a{" "} - - connection - {" "} - to start indexing repositories - - -
-
-
- ) -} - -function RepoSkeleton() { - return ( -
- {/* Skeleton for "Search X repositories" text */} -
- {/* "Search X" */} - {/* "repositories" */} -
- - {/* Skeleton for repository carousel */} - - - {[1, 2, 3].map((_, index) => ( - -
- {/* Icon */} - {/* Repository name */} -
- - ))} - - -
- ) -} diff --git a/packages/web/src/app/[domain]/components/navigationMenu.tsx b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx similarity index 56% rename from packages/web/src/app/[domain]/components/navigationMenu.tsx rename to packages/web/src/app/[domain]/components/navigationMenu/index.tsx index b71b7ab7..6cab2a68 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx @@ -1,21 +1,26 @@ +import { getRepos, getReposStats } from "@/actions"; +import { SourcebotLogo } from "@/app/components/sourcebotLogo"; +import { auth } from "@/auth"; +import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { NavigationMenu as NavigationMenuBase, NavigationMenuItem, NavigationMenuLink, NavigationMenuList, navigationMenuTriggerStyle } from "@/components/ui/navigation-menu"; -import Link from "next/link"; import { Separator } from "@/components/ui/separator"; -import { SettingsDropdown } from "./settingsDropdown"; -import { GitHubLogoIcon, DiscordLogoIcon } from "@radix-ui/react-icons"; -import { redirect } from "next/navigation"; -import { OrgSelector } from "./orgSelector"; -import { ErrorNavIndicator } from "./errorNavIndicator"; -import { WarningNavIndicator } from "./warningNavIndicator"; -import { ProgressNavIndicator } from "./progressNavIndicator"; -import { SourcebotLogo } from "@/app/components/sourcebotLogo"; -import { TrialNavIndicator } from "./trialNavIndicator"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { getSubscriptionInfo } from "@/ee/features/billing/actions"; import { IS_BILLING_ENABLED } from "@/ee/features/billing/stripe"; import { env } from "@/env.mjs"; -import { getSubscriptionInfo } from "@/ee/features/billing/actions"; -import { auth } from "@/auth"; -import WhatsNewIndicator from "./whatsNewIndicator"; +import { ServiceErrorException } from "@/lib/serviceError"; +import { cn, getShortenedNumberDisplayString, isServiceError } from "@/lib/utils"; +import { DiscordLogoIcon, GitHubLogoIcon } from "@radix-ui/react-icons"; +import { RepoJobStatus, RepoJobType } from "@sourcebot/db"; +import { BookMarkedIcon, CircleIcon, MessageCircleIcon, SearchIcon, SettingsIcon } from "lucide-react"; +import Link from "next/link"; +import { redirect } from "next/navigation"; +import { OrgSelector } from "../orgSelector"; +import { SettingsDropdown } from "../settingsDropdown"; +import WhatsNewIndicator from "../whatsNewIndicator"; +import { ProgressIndicator } from "./progressIndicator"; +import { TrialIndicator } from "./trialIndicator"; const SOURCEBOT_DISCORD_URL = "https://discord.gg/6Fhp27x7Pb"; const SOURCEBOT_GITHUB_URL = "https://github.com/sourcebot-dev/sourcebot"; @@ -31,6 +36,38 @@ export const NavigationMenu = async ({ const session = await auth(); const isAuthenticated = session?.user !== undefined; + const repoStats = await getReposStats(); + if (isServiceError(repoStats)) { + throw new ServiceErrorException(repoStats); + } + + const sampleRepos = await getRepos({ + where: { + jobs: { + some: { + type: RepoJobType.INDEX, + status: { + in: [ + RepoJobStatus.PENDING, + RepoJobStatus.IN_PROGRESS, + ] + } + }, + }, + indexedAt: null, + }, + take: 5, + }); + + if (isServiceError(sampleRepos)) { + throw new ServiceErrorException(sampleRepos); + } + + const { + numberOfRepos, + numberOfReposWithFirstTimeIndexingJobsInProgress, + } = repoStats; + return (
@@ -55,48 +92,55 @@ export const NavigationMenu = async ({ )} - + + Search - Repositories + + Ask - {isAuthenticated && ( - <> - {env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === undefined && ( - - - Agents - - - )} - + + + - Connections + + Repositories + + {getShortenedNumberDisplayString(numberOfRepos)} + {numberOfReposWithFirstTimeIndexingJobsInProgress > 0 && ( + + )} + - + + +

{numberOfRepos} total {numberOfRepos === 1 ? 'repository' : 'repositories'}

+ + + + {isAuthenticated && ( + <> + Settings @@ -107,10 +151,11 @@ export const NavigationMenu = async ({
- - - - + +
{ @@ -145,7 +190,5 @@ export const NavigationMenu = async ({
- - ) } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx b/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx new file mode 100644 index 00000000..cc5d5478 --- /dev/null +++ b/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx @@ -0,0 +1,118 @@ +'use client'; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Separator } from "@/components/ui/separator"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { useDomain } from "@/hooks/useDomain"; +import { RepositoryQuery } from "@/lib/types"; +import { getCodeHostInfoForRepo, getShortenedNumberDisplayString } from "@/lib/utils"; +import clsx from "clsx"; +import { FileIcon, Loader2Icon, RefreshCwIcon } from "lucide-react"; +import Image from "next/image"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useMemo } from "react"; + +interface ProgressIndicatorProps { + numberOfReposWithFirstTimeIndexingJobsInProgress: number; + sampleRepos: RepositoryQuery[]; +} + +export const ProgressIndicator = ({ + numberOfReposWithFirstTimeIndexingJobsInProgress: numRepos, + sampleRepos, +}: ProgressIndicatorProps) => { + const domain = useDomain(); + const router = useRouter(); + + if (numRepos === 0) { + return null; + } + + const numReposString = getShortenedNumberDisplayString(numRepos); + + return ( + + + + + + {numReposString} + + + + +
+

{`Syncing ${numReposString} ${numRepos === 1 ? 'repository' : 'repositories'}`}

+ +
+ +
+ {sampleRepos.map((repo) => ( + + ))} +
+ {numRepos > sampleRepos.length && ( +
+ + {`View ${numRepos - sampleRepos.length} more`} + +
+ )} + + + ) +} + +const RepoItem = ({ repo }: { repo: RepositoryQuery }) => { + + const { repoIcon, displayName } = useMemo(() => { + const info = getCodeHostInfoForRepo({ + name: repo.repoName, + codeHostType: repo.codeHostType, + displayName: repo.repoDisplayName, + webUrl: repo.webUrl, + }); + + if (info) { + return { + repoIcon: {info.codeHostName}, + displayName: info.displayName, + } + } + + return { + repoIcon: , + displayName: repo.repoName, + } + + + }, [repo.repoName, repo.codeHostType, repo.repoDisplayName, repo.webUrl]); + + + return ( + + {repoIcon} + + {displayName} + + + ) +} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/trialNavIndicator.tsx b/packages/web/src/app/[domain]/components/navigationMenu/trialIndicator.tsx similarity index 95% rename from packages/web/src/app/[domain]/components/trialNavIndicator.tsx rename to packages/web/src/app/[domain]/components/navigationMenu/trialIndicator.tsx index 70b13ecd..f7af06fc 100644 --- a/packages/web/src/app/[domain]/components/trialNavIndicator.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/trialIndicator.tsx @@ -13,7 +13,7 @@ interface Props { } | null | ServiceError; } -export const TrialNavIndicator = ({ subscription }: Props) => { +export const TrialIndicator = ({ subscription }: Props) => { const domain = useDomain(); const captureEvent = useCaptureEvent(); diff --git a/packages/web/src/app/[domain]/components/pathHeader.tsx b/packages/web/src/app/[domain]/components/pathHeader.tsx index 18806937..11b5bf1d 100644 --- a/packages/web/src/app/[domain]/components/pathHeader.tsx +++ b/packages/web/src/app/[domain]/components/pathHeader.tsx @@ -3,7 +3,7 @@ import { cn, getCodeHostInfoForRepo } from "@/lib/utils"; import { LaptopIcon } from "@radix-ui/react-icons"; import Image from "next/image"; -import { getBrowsePath } from "../browse/hooks/useBrowseNavigation"; +import { getBrowsePath } from "../browse/hooks/utils"; import { ChevronRight, MoreHorizontal } from "lucide-react"; import { useCallback, useState, useMemo, useRef, useEffect } from "react"; import { useToast } from "@/components/hooks/use-toast"; diff --git a/packages/web/src/app/[domain]/components/progressNavIndicator.tsx b/packages/web/src/app/[domain]/components/progressNavIndicator.tsx deleted file mode 100644 index f9e0d8cb..00000000 --- a/packages/web/src/app/[domain]/components/progressNavIndicator.tsx +++ /dev/null @@ -1,73 +0,0 @@ -"use client"; - -import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { useDomain } from "@/hooks/useDomain"; -import { env } from "@/env.mjs"; -import { unwrapServiceError } from "@/lib/utils"; -import { RepoIndexingStatus } from "@prisma/client"; -import { useQuery } from "@tanstack/react-query"; -import { Loader2Icon } from "lucide-react"; -import Link from "next/link"; -import { getRepos } from "@/app/api/(client)/client"; - -export const ProgressNavIndicator = () => { - const domain = useDomain(); - const captureEvent = useCaptureEvent(); - - const { data: inProgressRepos, isPending, isError } = useQuery({ - queryKey: ['repos'], - queryFn: () => unwrapServiceError(getRepos()), - select: (data) => data.filter(repo => repo.repoIndexingStatus === RepoIndexingStatus.IN_INDEX_QUEUE || repo.repoIndexingStatus === RepoIndexingStatus.INDEXING), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - if (isPending || isError || inProgressRepos.length === 0) { - return null; - } - - return ( - captureEvent('wa_progress_nav_pressed', {})} - > - - captureEvent('wa_progress_nav_hover', {})}> -
- - {inProgressRepos.length} -
- - -
-
-
-

Indexing in Progress

-
-

- The following repositories are currently being indexed: -

-
- { - inProgressRepos.slice(0, 10) - .map(item => ( -
- {item.repoName} -
- ) - )} - {inProgressRepos.length > 10 && ( -
- And {inProgressRepos.length - 10} more... -
- )} -
-
- - - - ); -}; \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/repositoryCarousel.tsx b/packages/web/src/app/[domain]/components/repositoryCarousel.tsx new file mode 100644 index 00000000..bd4f2d1a --- /dev/null +++ b/packages/web/src/app/[domain]/components/repositoryCarousel.tsx @@ -0,0 +1,158 @@ +'use client'; + +import { + Carousel, + CarouselContent, + CarouselItem, +} from "@/components/ui/carousel"; +import { captureEvent } from "@/hooks/useCaptureEvent"; +import { RepositoryQuery } from "@/lib/types"; +import { getCodeHostInfoForRepo } from "@/lib/utils"; +import { FileIcon } from "@radix-ui/react-icons"; +import clsx from "clsx"; +import Autoscroll from "embla-carousel-auto-scroll"; +import Image from "next/image"; +import Link from "next/link"; +import { getBrowsePath } from "../browse/hooks/utils"; +import { useDomain } from "@/hooks/useDomain"; + +interface RepositoryCarouselProps { + displayRepos: RepositoryQuery[]; + numberOfReposWithIndex: number; +} + +export function RepositoryCarousel({ + displayRepos, + numberOfReposWithIndex, +}: RepositoryCarouselProps) { + const domain = useDomain(); + + if (numberOfReposWithIndex === 0) { + return ( +
+ No repositories found + +
+
+ + <> + Create a{" "} + + connection + {" "} + to start indexing repositories + + +
+
+
+ ) + } + + return ( +
+ + {`${numberOfReposWithIndex} `} + + {numberOfReposWithIndex > 1 ? 'repositories' : 'repository'} + + {` indexed`} + + + + {displayRepos.map((repo, index) => ( + + + + ))} + + + {process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === "demo" && ( +

+ Interested in using Sourcebot on your code? Check out our{' '} + captureEvent('wa_demo_docs_link_pressed', {})} + > + docs + +

+ )} +
+ ) +} + +interface RepositoryBadgeProps { + repo: RepositoryQuery; +} + +const RepositoryBadge = ({ + repo +}: RepositoryBadgeProps) => { + const domain = useDomain(); + const { repoIcon, displayName } = (() => { + const info = getCodeHostInfoForRepo({ + codeHostType: repo.codeHostType, + name: repo.repoName, + displayName: repo.repoDisplayName, + webUrl: repo.webUrl, + }); + + if (info) { + return { + repoIcon: {info.codeHostName}, + displayName: info.displayName, + } + } + + return { + repoIcon: , + displayName: repo.repoName, + } + })(); + + return ( + + {repoIcon} + + {displayName} + + + ) +} diff --git a/packages/web/src/app/[domain]/components/homepage/toolbar.tsx b/packages/web/src/app/[domain]/components/searchModeSelector.tsx similarity index 83% rename from packages/web/src/app/[domain]/components/homepage/toolbar.tsx rename to packages/web/src/app/[domain]/components/searchModeSelector.tsx index cc9c65e0..e615ac8c 100644 --- a/packages/web/src/app/[domain]/components/homepage/toolbar.tsx +++ b/packages/web/src/app/[domain]/components/searchModeSelector.tsx @@ -1,13 +1,16 @@ 'use client'; import { KeyboardShortcutHint } from "@/app/components/keyboardShortcutHint"; +import { useDomain } from "@/hooks/useDomain"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Separator } from "@/components/ui/separator"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { cn } from "@/lib/utils"; -import { MessageCircleIcon, SearchIcon, TriangleAlert } from "lucide-react"; +import { MessageCircleIcon, SearchIcon } from "lucide-react"; import Link from "next/link"; -import { useState } from "react"; +import { useRouter } from "next/navigation"; +import { useCallback, useState } from "react"; +import { useHotkeys } from "react-hotkeys-hook"; export type SearchMode = "precise" | "agentic"; @@ -17,24 +20,47 @@ const AGENTIC_SEARCH_DOCS_URL = "https://docs.sourcebot.dev/docs/features/ask/ov export interface SearchModeSelectorProps { searchMode: SearchMode; - isAgenticSearchEnabled: boolean; - onSearchModeChange: (searchMode: SearchMode) => void; className?: string; } export const SearchModeSelector = ({ searchMode, - isAgenticSearchEnabled, - onSearchModeChange, className, }: SearchModeSelectorProps) => { + const domain = useDomain(); const [focusedSearchMode, setFocusedSearchMode] = useState(searchMode); + const router = useRouter(); + + const onSearchModeChanged = useCallback((value: SearchMode) => { + router.push(`/${domain}/${value === "precise" ? "search" : "chat"}`); + }, [domain, router]); + + useHotkeys("mod+i", (e) => { + e.preventDefault(); + onSearchModeChanged("agentic"); + }, { + enableOnFormTags: true, + enableOnContentEditable: true, + description: "Switch to agentic search", + }); + + useHotkeys("mod+p", (e) => { + e.preventDefault(); + onSearchModeChanged("precise"); + }, { + enableOnFormTags: true, + enableOnContentEditable: true, + description: "Switch to precise search", + }); + return (
setSearchQuery(e.target.value)} + onKeyDown={handleInputKeyDown} + className="border-0 focus-visible:ring-0 focus-visible:ring-offset-0 h-11" + /> +
+
+ {sortedSearchScopeItems.length === 0 ? ( +
+ No results found.
- )} - {sortedSearchScopeItems.map(({ item, isSelected }) => { - return ( - toggleItem(item)} - className="cursor-pointer" - > + ) : ( +
+ {!searchQuery && (
- + Select all
-
- -
-
- - {item.name} - - {item.type === 'reposet' && ( - - {item.repoCount} repo{item.repoCount === 1 ? '' : 's'} - + )} +
+ {isMounted && virtualizer.getVirtualItems().map((virtualItem) => { + const { item, isSelected } = sortedSearchScopeItems[virtualItem.index]; + const isHighlighted = virtualItem.index === highlightedIndex; + return ( +
toggleItem(item)} + onMouseEnter={() => setHighlightedIndex(virtualItem.index)} + className={cn( + "cursor-pointer absolute top-0 left-0 w-full flex items-center px-2 py-1.5 text-sm rounded-sm", + isHighlighted ? "bg-accent text-accent-foreground" : "hover:bg-accent hover:text-accent-foreground" )} + style={{ + transform: `translateY(${virtualItem.start}px)`, + }} + > +
+ +
+
+ +
+
+ + {item.name} + + {item.type === 'reposet' && ( + + {item.repoCount} repo{item.repoCount === 1 ? '' : 's'} + + )} +
+
+
-
-
- - ); - })} - - - {selectedSearchScopes.length > 0 && ( - <> - - - Clear - - - )} - - + ); + })} +
+
+ )} +
+ {selectedSearchScopes.length > 0 && ( + <> + +
+ Clear +
+ + )} +
+ + ); } From 88705f5e7e0634e025de3211f0e89a0935727d40 Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 16 Oct 2025 20:38:08 -0700 Subject: [PATCH 10/24] Add navigation indicators --- .../components/navigationMenu/index.tsx | 71 +++------------ .../navigationMenu/navigationItems.tsx | 90 +++++++++++++++++++ 2 files changed, 100 insertions(+), 61 deletions(-) create mode 100644 packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx diff --git a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx index 6cab2a68..400481c2 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx @@ -1,24 +1,22 @@ import { getRepos, getReposStats } from "@/actions"; import { SourcebotLogo } from "@/app/components/sourcebotLogo"; import { auth } from "@/auth"; -import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; -import { NavigationMenu as NavigationMenuBase, NavigationMenuItem, NavigationMenuLink, NavigationMenuList, navigationMenuTriggerStyle } from "@/components/ui/navigation-menu"; +import { NavigationMenu as NavigationMenuBase } from "@/components/ui/navigation-menu"; import { Separator } from "@/components/ui/separator"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { getSubscriptionInfo } from "@/ee/features/billing/actions"; import { IS_BILLING_ENABLED } from "@/ee/features/billing/stripe"; import { env } from "@/env.mjs"; import { ServiceErrorException } from "@/lib/serviceError"; -import { cn, getShortenedNumberDisplayString, isServiceError } from "@/lib/utils"; +import { isServiceError } from "@/lib/utils"; import { DiscordLogoIcon, GitHubLogoIcon } from "@radix-ui/react-icons"; import { RepoJobStatus, RepoJobType } from "@sourcebot/db"; -import { BookMarkedIcon, CircleIcon, MessageCircleIcon, SearchIcon, SettingsIcon } from "lucide-react"; import Link from "next/link"; import { redirect } from "next/navigation"; import { OrgSelector } from "../orgSelector"; import { SettingsDropdown } from "../settingsDropdown"; import WhatsNewIndicator from "../whatsNewIndicator"; +import { NavigationItems } from "./navigationItems"; import { ProgressIndicator } from "./progressIndicator"; import { TrialIndicator } from "./trialIndicator"; @@ -70,7 +68,7 @@ export const NavigationMenu = async ({ return (
-
+
- - - - - Search - - - - - - Ask - - - - - - - - Repositories - - {getShortenedNumberDisplayString(numberOfRepos)} - {numberOfReposWithFirstTimeIndexingJobsInProgress > 0 && ( - - )} - - - - -

{numberOfRepos} total {numberOfRepos === 1 ? 'repository' : 'repositories'}

- - - - {isAuthenticated && ( - <> - - - - Settings - - - - )} - +
diff --git a/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx b/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx new file mode 100644 index 00000000..055c8b10 --- /dev/null +++ b/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx @@ -0,0 +1,90 @@ +"use client"; + +import { NavigationMenuItem, NavigationMenuLink, NavigationMenuList, navigationMenuTriggerStyle } from "@/components/ui/navigation-menu"; +import { Badge } from "@/components/ui/badge"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { cn, getShortenedNumberDisplayString } from "@/lib/utils"; +import { SearchIcon, MessageCircleIcon, BookMarkedIcon, SettingsIcon, CircleIcon } from "lucide-react"; +import { usePathname } from "next/navigation"; + +interface NavigationItemsProps { + domain: string; + numberOfRepos: number; + numberOfReposWithFirstTimeIndexingJobsInProgress: number; + isAuthenticated: boolean; +} + +export const NavigationItems = ({ + domain, + numberOfRepos, + numberOfReposWithFirstTimeIndexingJobsInProgress, + isAuthenticated, +}: NavigationItemsProps) => { + const pathname = usePathname(); + + const isActive = (href: string) => { + if (href === `/${domain}`) { + return pathname === `/${domain}`; + } + return pathname.startsWith(href); + }; + + return ( + + + + + Search + + {isActive(`/${domain}`) && } + + + + + Ask + + {isActive(`/${domain}/chat`) && } + + + + + Repositories + + {getShortenedNumberDisplayString(numberOfRepos)} + {numberOfReposWithFirstTimeIndexingJobsInProgress > 0 && ( + + )} + + + {isActive(`/${domain}/repos`) && } + + {isAuthenticated && ( + + + + Settings + + {isActive(`/${domain}/settings`) && } + + )} + + ); +}; + +const ActiveIndicator = () => { + return ( +
+ ); +}; \ No newline at end of file From cfb359351d31f1cf04686548f3569d5222f61499 Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 16 Oct 2025 23:07:34 -0700 Subject: [PATCH 11/24] delete connections components --- .../[id]/components/connectionError.tsx | 110 ---- .../components/deleteConnectionSetting.tsx | 100 --- .../[id]/components/displayNameSetting.tsx | 100 --- .../[id]/components/notFoundWarning.tsx | 80 --- .../connections/[id]/components/overview.tsx | 159 ----- .../connections/[id]/components/repoList.tsx | 229 ------- .../[id]/components/repoListItem.tsx | 113 ---- .../[id]/components/repoListItemSkeleton.tsx | 15 - .../[id]/components/repoRetryIndexButton.tsx | 43 -- .../app/[domain]/connections/[id]/page.tsx | 63 -- .../connections/components/connectionIcon.tsx | 38 -- .../connectionList/connectionListItem.tsx | 120 ---- .../connectionListItemErrorIndicator.tsx | 62 -- .../connectionListItemManageButton.tsx | 37 -- .../connectionListItemWarningIndicator.tsx | 78 --- .../components/connectionList/index.tsx | 144 ----- .../connections/components/statusIcon.tsx | 29 - .../src/app/[domain]/connections/layout.tsx | 36 -- .../web/src/app/[domain]/connections/page.tsx | 35 -- .../app/[domain]/connections/quickActions.tsx | 575 ------------------ .../web/src/app/[domain]/connections/utils.ts | 40 -- 21 files changed, 2206 deletions(-) delete mode 100644 packages/web/src/app/[domain]/connections/[id]/components/connectionError.tsx delete mode 100644 packages/web/src/app/[domain]/connections/[id]/components/deleteConnectionSetting.tsx delete mode 100644 packages/web/src/app/[domain]/connections/[id]/components/displayNameSetting.tsx delete mode 100644 packages/web/src/app/[domain]/connections/[id]/components/notFoundWarning.tsx delete mode 100644 packages/web/src/app/[domain]/connections/[id]/components/overview.tsx delete mode 100644 packages/web/src/app/[domain]/connections/[id]/components/repoList.tsx delete mode 100644 packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx delete mode 100644 packages/web/src/app/[domain]/connections/[id]/components/repoListItemSkeleton.tsx delete mode 100644 packages/web/src/app/[domain]/connections/[id]/components/repoRetryIndexButton.tsx delete mode 100644 packages/web/src/app/[domain]/connections/[id]/page.tsx delete mode 100644 packages/web/src/app/[domain]/connections/components/connectionIcon.tsx delete mode 100644 packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx delete mode 100644 packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemErrorIndicator.tsx delete mode 100644 packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemManageButton.tsx delete mode 100644 packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemWarningIndicator.tsx delete mode 100644 packages/web/src/app/[domain]/connections/components/connectionList/index.tsx delete mode 100644 packages/web/src/app/[domain]/connections/components/statusIcon.tsx delete mode 100644 packages/web/src/app/[domain]/connections/layout.tsx delete mode 100644 packages/web/src/app/[domain]/connections/page.tsx delete mode 100644 packages/web/src/app/[domain]/connections/quickActions.tsx delete mode 100644 packages/web/src/app/[domain]/connections/utils.ts diff --git a/packages/web/src/app/[domain]/connections/[id]/components/connectionError.tsx b/packages/web/src/app/[domain]/connections/[id]/components/connectionError.tsx deleted file mode 100644 index 928cdca7..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/connectionError.tsx +++ /dev/null @@ -1,110 +0,0 @@ -"use client" - -import { BackendError } from "@sourcebot/error"; -import { Prisma } from "@sourcebot/db"; - -export function DisplayConnectionError({ syncStatusMetadata, onSecretsClick }: { syncStatusMetadata: Prisma.JsonValue, onSecretsClick: () => void }) { - const errorCode = syncStatusMetadata && typeof syncStatusMetadata === 'object' && 'error' in syncStatusMetadata - ? (syncStatusMetadata.error as string) - : undefined; - - switch (errorCode) { - case BackendError.CONNECTION_SYNC_INVALID_TOKEN: - return - case BackendError.CONNECTION_SYNC_SECRET_DNE: - return - case BackendError.CONNECTION_SYNC_SYSTEM_ERROR: - return - case BackendError.CONNECTION_SYNC_FAILED_TO_FETCH_GERRIT_PROJECTS: - return - default: - return - } -} - -function SecretNotFoundError({ syncStatusMetadata, onSecretsClick }: { syncStatusMetadata: Prisma.JsonValue, onSecretsClick: () => void }) { - const secretKey = syncStatusMetadata && typeof syncStatusMetadata === 'object' && 'secretKey' in syncStatusMetadata - ? (syncStatusMetadata.secretKey as string) - : undefined; - - return ( -
-

Secret Not Found

-

- The secret key provided for this connection was not found. Please ensure your config is referencing a secret - that exists in your{" "} - - , and try again. -

- {secretKey && ( -

- Secret Key: {secretKey} -

- )} -
- ); -} - -function InvalidTokenError({ syncStatusMetadata, onSecretsClick }: { syncStatusMetadata: Prisma.JsonValue, onSecretsClick: () => void }) { - const secretKey = syncStatusMetadata && typeof syncStatusMetadata === 'object' && 'secretKey' in syncStatusMetadata - ? (syncStatusMetadata.secretKey as string) - : undefined; - - return ( -
-

Invalid Authentication Token

-

- The authentication token provided for this connection is invalid. Please update your config with a valid token and try again. -

- {secretKey && ( -

- Secret Key: -

- )} -
- ); -} - -function SystemError() { - return ( -
-

System Error

-

- An error occurred while syncing this connection. Please try again later. -

-
- ) -} - -function FailedToFetchGerritProjects({ syncStatusMetadata }: { syncStatusMetadata: Prisma.JsonValue}) { - const status = syncStatusMetadata && typeof syncStatusMetadata === 'object' && 'status' in syncStatusMetadata - ? (syncStatusMetadata.status as number) - : undefined; - - return ( -
-

Failed to Fetch Gerrit Projects

-

- An error occurred while syncing this connection. Please try again later. -

- {status && ( -

- Status: {status} -

- )} -
- ) -} - -function UnknownError() { - return ( -
-

Unknown Error

-

- An unknown error occurred while syncing this connection. Please try again later. -

-
- ) -} diff --git a/packages/web/src/app/[domain]/connections/[id]/components/deleteConnectionSetting.tsx b/packages/web/src/app/[domain]/connections/[id]/components/deleteConnectionSetting.tsx deleted file mode 100644 index 1eb2488d..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/deleteConnectionSetting.tsx +++ /dev/null @@ -1,100 +0,0 @@ -'use client'; - -import { Button } from "@/components/ui/button"; -import { useCallback, useState } from "react"; -import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, - AlertDialogTrigger, - } from "@/components/ui/alert-dialog"; -import { deleteConnection } from "@/actions"; -import { Loader2 } from "lucide-react"; -import { isServiceError } from "@/lib/utils"; -import { useToast } from "@/components/hooks/use-toast"; -import { useRouter } from "next/navigation"; -import { useDomain } from "@/hooks/useDomain"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; - -interface DeleteConnectionSettingProps { - connectionId: number; - disabled?: boolean; -} - -export const DeleteConnectionSetting = ({ - connectionId, - disabled, -}: DeleteConnectionSettingProps) => { - const [isDialogOpen, setIsDialogOpen] = useState(false); - const [isLoading, setIsLoading] = useState(false); - const domain = useDomain(); - const { toast } = useToast(); - const router = useRouter(); - const captureEvent = useCaptureEvent(); - - const handleDelete = useCallback(() => { - setIsDialogOpen(false); - setIsLoading(true); - deleteConnection(connectionId, domain) - .then((response) => { - if (isServiceError(response)) { - toast({ - description: `❌ Failed to delete connection. Reason: ${response.message}` - }); - captureEvent('wa_connection_delete_fail', { - error: response.errorCode, - }); - } else { - toast({ - description: `✅ Connection deleted successfully.` - }); - captureEvent('wa_connection_delete_success', {}); - router.replace(`/${domain}/connections`); - router.refresh(); - } - }) - .finally(() => { - setIsLoading(false); - }); - }, [connectionId, domain, router, toast, captureEvent]); - - return ( -
-

Delete Connection

-

- Permanently delete this connection from Sourcebot. All linked repositories that are not linked to any other connection will also be deleted. -

-
- - - - - - - Are you sure? - - This action cannot be undone. - - - - Cancel - Yes, delete connection - - - -
-
- ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/[id]/components/displayNameSetting.tsx b/packages/web/src/app/[domain]/connections/[id]/components/displayNameSetting.tsx deleted file mode 100644 index e7f795e8..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/displayNameSetting.tsx +++ /dev/null @@ -1,100 +0,0 @@ -'use client'; - -import { updateConnectionDisplayName } from "@/actions"; -import { useToast } from "@/components/hooks/use-toast"; -import { Button } from "@/components/ui/button"; -import { Form, FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; -import { useDomain } from "@/hooks/useDomain"; -import { isServiceError } from "@/lib/utils"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { Loader2 } from "lucide-react"; -import { useRouter } from "next/navigation"; -import { useCallback, useState } from "react"; -import { useForm } from "react-hook-form"; -import { z } from "zod"; - -const formSchema = z.object({ - name: z.string().min(1), -}); - -interface DisplayNameSettingProps { - connectionId: number; - name: string; - disabled?: boolean; -} - -export const DisplayNameSetting = ({ - connectionId, - name, - disabled, -}: DisplayNameSettingProps) => { - const { toast } = useToast(); - const router = useRouter(); - const domain = useDomain(); - const form = useForm>({ - resolver: zodResolver(formSchema), - defaultValues: { - name, - }, - }); - - const [isLoading, setIsLoading] = useState(false); - const onSubmit = useCallback((data: z.infer) => { - setIsLoading(true); - updateConnectionDisplayName(connectionId, data.name, domain) - .then((response) => { - if (isServiceError(response)) { - toast({ - description: `❌ Failed to rename connection. Reason: ${response.message}` - }); - } else { - toast({ - description: `✅ Connection renamed successfully.` - }); - router.refresh(); - } - }).finally(() => { - setIsLoading(false); - }); - }, [connectionId, domain, router, toast]); - - return ( -
- - - ( - - Display Name - {/* @todo : refactor this description into a shared file */} - This is the {`connection's`} display name within Sourcebot. Examples: public-github, self-hosted-gitlab, gerrit-other, etc. - - - - - - )} - /> -
- -
- - -
- ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/[id]/components/notFoundWarning.tsx b/packages/web/src/app/[domain]/connections/[id]/components/notFoundWarning.tsx deleted file mode 100644 index c65d4f10..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/notFoundWarning.tsx +++ /dev/null @@ -1,80 +0,0 @@ -'use client'; - -import { AlertTriangle } from "lucide-react" -import { Prisma, ConnectionSyncStatus } from "@sourcebot/db" -import { SyncStatusMetadataSchema } from "@/lib/syncStatusMetadataSchema" -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { ReloadIcon } from "@radix-ui/react-icons"; -import { Button } from "@/components/ui/button"; - -interface NotFoundWarningProps { - syncStatus: ConnectionSyncStatus - syncStatusMetadata: Prisma.JsonValue - onSecretsClick: () => void - connectionType: string - onRetrySync: () => void -} - -export const NotFoundWarning = ({ syncStatus, syncStatusMetadata, onSecretsClick, connectionType, onRetrySync }: NotFoundWarningProps) => { - const captureEvent = useCaptureEvent(); - - const parseResult = SyncStatusMetadataSchema.safeParse(syncStatusMetadata); - if (syncStatus !== ConnectionSyncStatus.SYNCED_WITH_WARNINGS || !parseResult.success || !parseResult.data.notFound) { - return null; - } - - const { notFound } = parseResult.data; - - if (notFound.users.length === 0 && notFound.orgs.length === 0 && notFound.repos.length === 0) { - return null; - } else { - captureEvent('wa_connection_not_found_warning_displayed', {}); - } - - return ( -
-
- -

Unable to fetch all references

-
-

- Some requested references couldn't be found. Please ensure you've provided the information listed below correctly, and that you've provided a{" "} - {" "} - to access them if they're private. -

-
    - {notFound.users.length > 0 && ( -
  • - Users: - {notFound.users.join(', ')} -
    • - )} - {notFound.orgs.length > 0 && ( -
  • - {connectionType === "gitlab" ? "Groups" : "Organizations"}: - {notFound.orgs.join(', ')} -
    • - )} - {notFound.repos.length > 0 && ( -
  • - {connectionType === "gitlab" ? "Projects" : "Repositories"}: - {notFound.repos.join(', ')} -
    • - )} -
-
- -
-
- ) -} diff --git a/packages/web/src/app/[domain]/connections/[id]/components/overview.tsx b/packages/web/src/app/[domain]/connections/[id]/components/overview.tsx deleted file mode 100644 index 6b7dbbaf..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/overview.tsx +++ /dev/null @@ -1,159 +0,0 @@ -'use client'; - -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card" -import { DisplayConnectionError } from "./connectionError" -import { NotFoundWarning } from "./notFoundWarning" -import { useDomain } from "@/hooks/useDomain"; -import { useRouter } from "next/navigation"; -import { useCallback } from "react"; -import { useQuery } from "@tanstack/react-query"; -import { flagConnectionForSync, getConnectionInfo } from "@/actions"; -import { isServiceError, unwrapServiceError } from "@/lib/utils"; -import { env } from "@/env.mjs"; -import { ConnectionSyncStatus } from "@sourcebot/db"; -import { FiLoader } from "react-icons/fi"; -import { CircleCheckIcon, AlertTriangle, CircleXIcon } from "lucide-react"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { ReloadIcon } from "@radix-ui/react-icons"; -import { toast } from "@/components/hooks/use-toast"; - -interface OverviewProps { - connectionId: number; -} - -export const Overview = ({ connectionId }: OverviewProps) => { - const captureEvent = useCaptureEvent(); - const domain = useDomain(); - const router = useRouter(); - - const { data: connection, isPending, error, refetch } = useQuery({ - queryKey: ['connection', domain, connectionId], - queryFn: () => unwrapServiceError(getConnectionInfo(connectionId, domain)), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - const handleSecretsNavigation = useCallback(() => { - captureEvent('wa_connection_secrets_navigation_pressed', {}); - router.push(`/${domain}/secrets`); - }, [captureEvent, domain, router]); - - const onRetrySync = useCallback(async () => { - const result = await flagConnectionForSync(connectionId, domain); - if (isServiceError(result)) { - toast({ - description: `❌ Failed to flag connection for sync.`, - }); - captureEvent('wa_connection_retry_sync_fail', { - error: result.errorCode, - }); - } else { - toast({ - description: "✅ Connection flagged for sync.", - }); - captureEvent('wa_connection_retry_sync_success', {}); - refetch(); - } - }, [connectionId, domain, captureEvent, refetch]); - - - if (error) { - return
- {`Error loading connection. Reason: ${error.message}`} -
- } - - if (isPending) { - return ( -
- {Array.from({ length: 4 }).map((_, i) => ( -
-
-
-
- ))} -
- ) - } - - return ( -
-
-
-

Connection Type

-

{connection.connectionType}

-
-
-

Last Synced At

-

- {connection.syncedAt ? new Date(connection.syncedAt).toLocaleDateString() : "never"} -

-
-
-

Linked Repositories

-

{connection.numLinkedRepos}

-
-
-

Status

-
- {connection.syncStatus === "FAILED" ? ( - - captureEvent('wa_connection_failed_status_hover', {})}> - - - - - - - ) : ( - - )} - {connection.syncStatus === "FAILED" && ( - - )} -
-
-
- -
- ) -} - -const SyncStatusBadge = ({ status }: { status: ConnectionSyncStatus }) => { - return ( - - {status === ConnectionSyncStatus.SYNC_NEEDED || status === ConnectionSyncStatus.IN_SYNC_QUEUE ? ( - <> Sync queued - ) : status === ConnectionSyncStatus.SYNCING ? ( - <> Syncing - ) : status === ConnectionSyncStatus.SYNCED ? ( - Synced - ) : status === ConnectionSyncStatus.SYNCED_WITH_WARNINGS ? ( - Synced with warnings - ) : status === ConnectionSyncStatus.FAILED ? ( - <> Sync failed - ) : null} - - ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/[id]/components/repoList.tsx b/packages/web/src/app/[domain]/connections/[id]/components/repoList.tsx deleted file mode 100644 index cab5c4b1..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/repoList.tsx +++ /dev/null @@ -1,229 +0,0 @@ -'use client'; - -import { useDomain } from "@/hooks/useDomain"; -import { useQuery } from "@tanstack/react-query"; -import { flagReposForIndex, getConnectionInfo, getRepos } from "@/actions"; -import { RepoListItem } from "./repoListItem"; -import { isServiceError, unwrapServiceError } from "@/lib/utils"; -import { ScrollArea } from "@/components/ui/scroll-area"; -import { ConnectionSyncStatus, RepoIndexingStatus } from "@sourcebot/db"; -import { Search, Loader2 } from "lucide-react"; -import { Input } from "@/components/ui/input"; -import { useCallback, useMemo, useState } from "react"; -import { RepoListItemSkeleton } from "./repoListItemSkeleton"; -import { env } from "@/env.mjs"; -import { Button } from "@/components/ui/button"; -import { useRouter } from "next/navigation"; -import { MultiSelect } from "@/components/ui/multi-select"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { useToast } from "@/components/hooks/use-toast"; - -interface RepoListProps { - connectionId: number; -} - -const getPriority = (status: RepoIndexingStatus) => { - switch (status) { - case RepoIndexingStatus.FAILED: - return 0 - case RepoIndexingStatus.IN_INDEX_QUEUE: - case RepoIndexingStatus.INDEXING: - return 1 - case RepoIndexingStatus.INDEXED: - return 2 - default: - return 3 - } -} - -const convertIndexingStatus = (status: RepoIndexingStatus) => { - switch (status) { - case RepoIndexingStatus.FAILED: - return 'failed'; - case RepoIndexingStatus.NEW: - return 'waiting'; - case RepoIndexingStatus.IN_INDEX_QUEUE: - case RepoIndexingStatus.INDEXING: - return 'running'; - case RepoIndexingStatus.INDEXED: - return 'succeeded'; - default: - return 'unknown'; - } -} - -export const RepoList = ({ connectionId }: RepoListProps) => { - const domain = useDomain(); - const router = useRouter(); - const { toast } = useToast(); - const [searchQuery, setSearchQuery] = useState(""); - const [selectedStatuses, setSelectedStatuses] = useState([]); - const captureEvent = useCaptureEvent(); - const [isRetryAllFailedReposLoading, setIsRetryAllFailedReposLoading] = useState(false); - - const { data: unfilteredRepos, isPending: isReposPending, error: reposError, refetch: refetchRepos } = useQuery({ - queryKey: ['repos', domain, connectionId], - queryFn: async () => { - const repos = await unwrapServiceError(getRepos()); - return repos.sort((a, b) => { - const priorityA = getPriority(a.repoIndexingStatus); - const priorityB = getPriority(b.repoIndexingStatus); - - // First sort by priority - if (priorityA !== priorityB) { - return priorityA - priorityB; - } - - // If same priority, sort by indexedAt - return new Date(a.indexedAt ?? new Date()).getTime() - new Date(b.indexedAt ?? new Date()).getTime(); - }); - }, - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - const { data: connection, isPending: isConnectionPending, error: isConnectionError } = useQuery({ - queryKey: ['connection', domain, connectionId], - queryFn: () => unwrapServiceError(getConnectionInfo(connectionId, domain)), - }) - - - const failedRepos = useMemo(() => { - return unfilteredRepos?.filter((repo) => repo.repoIndexingStatus === RepoIndexingStatus.FAILED) ?? []; - }, [unfilteredRepos]); - - - const onRetryAllFailedRepos = useCallback(() => { - if (failedRepos.length === 0) { - return; - } - - setIsRetryAllFailedReposLoading(true); - flagReposForIndex(failedRepos.map((repo) => repo.repoId)) - .then((response) => { - if (isServiceError(response)) { - captureEvent('wa_connection_retry_all_failed_repos_fail', {}); - toast({ - description: `❌ Failed to flag repositories for indexing. Reason: ${response.message}`, - }); - } else { - captureEvent('wa_connection_retry_all_failed_repos_success', {}); - toast({ - description: `✅ ${failedRepos.length} repositories flagged for indexing.`, - }); - } - }) - .then(() => { refetchRepos() }) - .finally(() => { - setIsRetryAllFailedReposLoading(false); - }); - }, [captureEvent, failedRepos, refetchRepos, toast]); - - const filteredRepos = useMemo(() => { - if (isServiceError(unfilteredRepos)) { - return unfilteredRepos; - } - - const searchLower = searchQuery.toLowerCase(); - return unfilteredRepos?.filter((repo) => { - return repo.repoName.toLowerCase().includes(searchLower); - }).filter((repo) => { - if (selectedStatuses.length === 0) { - return true; - } - - return selectedStatuses.includes(convertIndexingStatus(repo.repoIndexingStatus)); - }); - }, [unfilteredRepos, searchQuery, selectedStatuses]); - - if (reposError) { - return
- {`Error loading repositories. Reason: ${reposError.message}`} -
- } - - return ( -
-
-
- - setSearchQuery(e.target.value)} - /> -
- - setSelectedStatuses(value)} - defaultValue={[]} - placeholder="Filter by status" - maxCount={2} - animation={0} - /> - - {failedRepos.length > 0 && ( - - )} -
- - {isReposPending ? ( -
- {Array.from({ length: 3 }).map((_, i) => ( - - ))} -
- ) : (!filteredRepos || filteredRepos.length === 0) ? ( -
-

No Repositories Found

-

- { - searchQuery.length > 0 ? ( - No repositories found matching your filters. - ) : (!isConnectionError && !isConnectionPending && (connection.syncStatus === ConnectionSyncStatus.IN_SYNC_QUEUE || connection.syncStatus === ConnectionSyncStatus.SYNCING || connection.syncStatus === ConnectionSyncStatus.SYNC_NEEDED)) ? ( - Repositories are being synced. Please check back soon. - ) : ( - - )} -

-
- ) : ( -
- {filteredRepos?.map((repo) => ( - - ))} -
- )} - -
- ) -} diff --git a/packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx b/packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx deleted file mode 100644 index fd491376..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx +++ /dev/null @@ -1,113 +0,0 @@ -'use client'; - -import { getDisplayTime, getRepoImageSrc } from "@/lib/utils"; -import Image from "next/image"; -import { StatusIcon } from "../../components/statusIcon"; -import { RepoIndexingStatus } from "@sourcebot/db"; -import { useMemo } from "react"; -import { RetryRepoIndexButton } from "./repoRetryIndexButton"; - - -interface RepoListItemProps { - name: string; - status: RepoIndexingStatus; - imageUrl?: string; - indexedAt?: Date; - repoId: number; - domain: string; -} - -export const RepoListItem = ({ - imageUrl, - name, - indexedAt, - status, - repoId, - domain, -}: RepoListItemProps) => { - const statusDisplayName = useMemo(() => { - switch (status) { - case RepoIndexingStatus.NEW: - return 'Waiting...'; - case RepoIndexingStatus.IN_INDEX_QUEUE: - return 'In index queue...'; - case RepoIndexingStatus.INDEXING: - return 'Indexing...'; - case RepoIndexingStatus.INDEXED: - return 'Indexed'; - case RepoIndexingStatus.FAILED: - return 'Index failed'; - case RepoIndexingStatus.IN_GC_QUEUE: - return 'In garbage collection queue...'; - case RepoIndexingStatus.GARBAGE_COLLECTING: - return 'Garbage collecting...'; - case RepoIndexingStatus.GARBAGE_COLLECTION_FAILED: - return 'Garbage collection failed'; - } - }, [status]); - - const imageSrc = getRepoImageSrc(imageUrl, repoId, domain); - - return ( -
-
- {imageSrc ? ( - {name} - ) : ( -
- {name.charAt(0)} -
- )} -

{name}

-
-
- {status === RepoIndexingStatus.FAILED && ( - - )} -
- -

- {statusDisplayName} - { - ( - status === RepoIndexingStatus.INDEXED || - status === RepoIndexingStatus.FAILED - ) && indexedAt && ( - {` ${getDisplayTime(indexedAt)}`} - ) - } -

-
-
-
- ) -} - -const convertIndexingStatus = (status: RepoIndexingStatus) => { - switch (status) { - case RepoIndexingStatus.NEW: - return 'waiting'; - case RepoIndexingStatus.IN_INDEX_QUEUE: - case RepoIndexingStatus.INDEXING: - return 'running'; - case RepoIndexingStatus.IN_GC_QUEUE: - case RepoIndexingStatus.GARBAGE_COLLECTING: - return "garbage-collecting" - case RepoIndexingStatus.INDEXED: - return 'succeeded'; - case RepoIndexingStatus.GARBAGE_COLLECTION_FAILED: - case RepoIndexingStatus.FAILED: - return 'failed'; - } -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/[id]/components/repoListItemSkeleton.tsx b/packages/web/src/app/[domain]/connections/[id]/components/repoListItemSkeleton.tsx deleted file mode 100644 index 2eab9a38..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/repoListItemSkeleton.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import { Skeleton } from "@/components/ui/skeleton" - -export const RepoListItemSkeleton = () => { - return ( -
-
- - -
-
- -
-
- ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/[id]/components/repoRetryIndexButton.tsx b/packages/web/src/app/[domain]/connections/[id]/components/repoRetryIndexButton.tsx deleted file mode 100644 index ad44c60e..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/repoRetryIndexButton.tsx +++ /dev/null @@ -1,43 +0,0 @@ -"use client"; - -import { Button } from "@/components/ui/button"; -import { ReloadIcon } from "@radix-ui/react-icons" -import { toast } from "@/components/hooks/use-toast"; -import { flagReposForIndex } from "@/actions"; -import { isServiceError } from "@/lib/utils"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; - -interface RetryRepoIndexButtonProps { - repoId: number; -} - -export const RetryRepoIndexButton = ({ repoId }: RetryRepoIndexButtonProps) => { - const captureEvent = useCaptureEvent(); - - return ( - - ); -}; diff --git a/packages/web/src/app/[domain]/connections/[id]/page.tsx b/packages/web/src/app/[domain]/connections/[id]/page.tsx deleted file mode 100644 index 0e0a91d2..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/page.tsx +++ /dev/null @@ -1,63 +0,0 @@ -import { NotFound } from "@/app/[domain]/components/notFound" -import { - Breadcrumb, - BreadcrumbItem, - BreadcrumbLink, - BreadcrumbList, - BreadcrumbPage, - BreadcrumbSeparator, -} from "@/components/ui/breadcrumb" -import { ConnectionIcon } from "../components/connectionIcon" -import { Header } from "../../components/header" -import { RepoList } from "./components/repoList" -import { getConnectionByDomain } from "@/data/connection" -import { Overview } from "./components/overview" - -interface ConnectionManagementPageProps { - params: Promise<{ - domain: string - id: string - }>, -} - -export default async function ConnectionManagementPage(props: ConnectionManagementPageProps) { - const params = await props.params; - const connection = await getConnectionByDomain(Number(params.id), params.domain); - if (!connection) { - return - } - - return ( -
-
- - - - Connections - - - - {connection.name} - - - -
- -

{connection.name}

-
-
- -
-
-

Overview

- -
- -
-

Linked Repositories

- -
-
-
- ) -} diff --git a/packages/web/src/app/[domain]/connections/components/connectionIcon.tsx b/packages/web/src/app/[domain]/connections/components/connectionIcon.tsx deleted file mode 100644 index 2c08121a..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionIcon.tsx +++ /dev/null @@ -1,38 +0,0 @@ -'use client'; - -import { cn, CodeHostType, getCodeHostIcon } from "@/lib/utils"; -import { useMemo } from "react"; -import Image from "next/image"; -import placeholderLogo from "@/public/placeholder_avatar.png"; - -interface ConnectionIconProps { - type: string; - className?: string; -} - -export const ConnectionIcon = ({ - type, - className, -}: ConnectionIconProps) => { - const Icon = useMemo(() => { - const iconInfo = getCodeHostIcon(type as CodeHostType); - if (iconInfo) { - return ( - {`${type} - ) - } - - return {''} - - }, [className, type]); - - return Icon; -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx deleted file mode 100644 index 02c41139..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx +++ /dev/null @@ -1,120 +0,0 @@ -import { getDisplayTime } from "@/lib/utils"; -import { useMemo } from "react"; -import { ConnectionIcon } from "../connectionIcon"; -import { ConnectionSyncStatus, Prisma } from "@sourcebot/db"; -import { StatusIcon } from "../statusIcon"; -import { ConnectionListItemErrorIndicator } from "./connectionListItemErrorIndicator"; -import { ConnectionListItemWarningIndicator } from "./connectionListItemWarningIndicator"; -import { ConnectionListItemManageButton } from "./connectionListItemManageButton"; - -const convertSyncStatus = (status: ConnectionSyncStatus) => { - switch (status) { - case ConnectionSyncStatus.SYNC_NEEDED: - return 'waiting'; - case ConnectionSyncStatus.IN_SYNC_QUEUE: - case ConnectionSyncStatus.SYNCING: - return 'running'; - case ConnectionSyncStatus.SYNCED: - return 'succeeded'; - case ConnectionSyncStatus.SYNCED_WITH_WARNINGS: - return 'succeeded-with-warnings'; - case ConnectionSyncStatus.FAILED: - return 'failed'; - } -} - -interface ConnectionListItemProps { - id: string; - name: string; - type: string; - status: ConnectionSyncStatus; - syncStatusMetadata: Prisma.JsonValue; - editedAt: Date; - syncedAt?: Date; - failedRepos?: { repoId: number, repoName: string }[]; - disabled: boolean; -} - -export const ConnectionListItem = ({ - id, - name, - type, - status, - syncStatusMetadata, - editedAt, - syncedAt, - failedRepos, - disabled, -}: ConnectionListItemProps) => { - const statusDisplayName = useMemo(() => { - switch (status) { - case ConnectionSyncStatus.SYNC_NEEDED: - return 'Waiting...'; - case ConnectionSyncStatus.IN_SYNC_QUEUE: - case ConnectionSyncStatus.SYNCING: - return 'Syncing...'; - case ConnectionSyncStatus.SYNCED: - return 'Synced'; - case ConnectionSyncStatus.FAILED: - return 'Sync failed'; - case ConnectionSyncStatus.SYNCED_WITH_WARNINGS: - return null; - } - }, [status]); - - const { notFoundData, displayNotFoundWarning } = useMemo(() => { - if (!syncStatusMetadata || typeof syncStatusMetadata !== 'object' || !('notFound' in syncStatusMetadata)) { - return { notFoundData: null, displayNotFoundWarning: false }; - } - - const notFoundData = syncStatusMetadata.notFound as { - users: string[], - orgs: string[], - repos: string[], - } - - return { notFoundData, displayNotFoundWarning: notFoundData.users.length > 0 || notFoundData.orgs.length > 0 || notFoundData.repos.length > 0 }; - }, [syncStatusMetadata]); - - return ( -
-
- -
-

{name}

- {`Edited ${getDisplayTime(editedAt)}`} -
- - -
-
- -

- {statusDisplayName} - { - ( - status === ConnectionSyncStatus.SYNCED || - status === ConnectionSyncStatus.FAILED - ) && syncedAt && ( - {` ${getDisplayTime(syncedAt)}`} - ) - } -

- -
-
- ) -} diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemErrorIndicator.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemErrorIndicator.tsx deleted file mode 100644 index 26c9ee21..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemErrorIndicator.tsx +++ /dev/null @@ -1,62 +0,0 @@ -'use client' - -import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; -import { CircleX } from "lucide-react"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; - -interface ConnectionListItemErrorIndicatorProps { - failedRepos: { repoId: number; repoName: string; }[] | undefined; - connectionId: string; -} - -export const ConnectionListItemErrorIndicator = ({ - failedRepos, - connectionId -}: ConnectionListItemErrorIndicatorProps) => { - const captureEvent = useCaptureEvent() - - if (!failedRepos || failedRepos.length === 0) return null; - - return ( - - - { - captureEvent('wa_connection_list_item_error_pressed', {}) - window.location.href = `connections/${connectionId}` - }} - onMouseEnter={() => captureEvent('wa_connection_list_item_error_hover', {})} - /> - - -
-
- -

Failed to Index Repositories

-
-
-

- {failedRepos.length} {failedRepos.length === 1 ? 'repository' : 'repositories'} failed to index. This is likely due to temporary server load. -

-
-
- {failedRepos.slice(0, 10).map(repo => ( - {repo.repoName} - ))} - {failedRepos.length > 10 && ( - - And {failedRepos.length - 10} more... - - )} -
-
-

- Navigate to the connection for more details and to retry indexing. -

-
-
- - - ); -}; diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemManageButton.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemManageButton.tsx deleted file mode 100644 index 00b7db27..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemManageButton.tsx +++ /dev/null @@ -1,37 +0,0 @@ -'use client' - -import { Button } from "@/components/ui/button"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { useRouter } from "next/navigation"; -import { useDomain } from "@/hooks/useDomain"; - -interface ConnectionListItemManageButtonProps { - id: string; - disabled: boolean; -} - -export const ConnectionListItemManageButton = ({ - id, - disabled, -}: ConnectionListItemManageButtonProps) => { - const captureEvent = useCaptureEvent() - const router = useRouter(); - const domain = useDomain(); - - return ( - - ); -}; diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemWarningIndicator.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemWarningIndicator.tsx deleted file mode 100644 index 690c9b18..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemWarningIndicator.tsx +++ /dev/null @@ -1,78 +0,0 @@ -'use client' - -import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; -import { AlertTriangle } from "lucide-react"; -import { NotFoundData } from "@/lib/syncStatusMetadataSchema"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; - - -interface ConnectionListItemWarningIndicatorProps { - notFoundData: NotFoundData | null; - connectionId: string; - type: string; - displayWarning: boolean; -} - -export const ConnectionListItemWarningIndicator = ({ - notFoundData, - connectionId, - type, - displayWarning -}: ConnectionListItemWarningIndicatorProps) => { - const captureEvent = useCaptureEvent() - - if (!notFoundData || !displayWarning) return null; - - return ( - - - { - captureEvent('wa_connection_list_item_warning_pressed', {}) - window.location.href = `connections/${connectionId}` - }} - onMouseEnter={() => captureEvent('wa_connection_list_item_warning_hover', {})} - /> - - -
-
- -

Unable to fetch all references

-
-

- Some requested references couldn't be found. Verify the details below and ensure your connection is using a {" "} - {" "} - that has access to any private references. -

-
    - {notFoundData.users.length > 0 && ( -
  • - Users: - {notFoundData.users.join(', ')} -
    • - )} - {notFoundData.orgs.length > 0 && ( -
  • - {type === "gitlab" ? "Groups" : "Organizations"}: - {notFoundData.orgs.join(', ')} -
    • - )} - {notFoundData.repos.length > 0 && ( -
  • - {type === "gitlab" ? "Projects" : "Repositories"}: - {notFoundData.repos.join(', ')} -
    • - )} -
-
- - - ); -}; \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/index.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/index.tsx deleted file mode 100644 index 94b4f8be..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionList/index.tsx +++ /dev/null @@ -1,144 +0,0 @@ -"use client"; -import { useDomain } from "@/hooks/useDomain"; -import { ConnectionListItem } from "./connectionListItem"; -import { cn, unwrapServiceError } from "@/lib/utils"; -import { InfoCircledIcon } from "@radix-ui/react-icons"; -import { getConnections } from "@/actions"; -import { Skeleton } from "@/components/ui/skeleton"; -import { useQuery } from "@tanstack/react-query"; -import { env } from "@/env.mjs"; -import { RepoIndexingStatus, ConnectionSyncStatus } from "@sourcebot/db"; -import { Search } from "lucide-react"; -import { Input } from "@/components/ui/input"; -import { useMemo, useState } from "react"; -import { MultiSelect } from "@/components/ui/multi-select"; - -interface ConnectionListProps { - className?: string; - isDisabled: boolean; -} - -const convertSyncStatus = (status: ConnectionSyncStatus) => { - switch (status) { - case ConnectionSyncStatus.SYNC_NEEDED: - return 'waiting'; - case ConnectionSyncStatus.SYNCING: - return 'running'; - case ConnectionSyncStatus.SYNCED: - return 'succeeded'; - case ConnectionSyncStatus.SYNCED_WITH_WARNINGS: - return 'synced-with-warnings'; - case ConnectionSyncStatus.FAILED: - return 'failed'; - default: - return 'unknown'; - } -} - -export const ConnectionList = ({ - className, - isDisabled, -}: ConnectionListProps) => { - const domain = useDomain(); - const [searchQuery, setSearchQuery] = useState(""); - const [selectedStatuses, setSelectedStatuses] = useState([]); - - const { data: unfilteredConnections, isPending, error } = useQuery({ - queryKey: ['connections', domain], - queryFn: () => unwrapServiceError(getConnections(domain)), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - const connections = useMemo(() => { - return unfilteredConnections - ?.filter((connection) => connection.name.toLowerCase().includes(searchQuery.toLowerCase())) - .filter((connection) => { - if (selectedStatuses.length === 0) { - return true; - } - - return selectedStatuses.includes(convertSyncStatus(connection.syncStatus)); - }) - .sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime()) ?? []; - }, [unfilteredConnections, searchQuery, selectedStatuses]); - - if (error) { - return
-

Error loading connections: {error.message}

-
- } - - return ( -
-
-
- - setSearchQuery(e.target.value)} - /> -
- - setSelectedStatuses(value)} - defaultValue={[]} - placeholder="Filter by status" - maxCount={2} - animation={0} - /> - -
- - {isPending ? ( - // Skeleton for loading state -
- {Array.from({ length: 3 }).map((_, i) => ( -
- -
- - -
- -
- ))} -
- ) : connections.length > 0 ? ( - connections - .sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime()) - .map((connection) => ( - repo.repoIndexingStatus === RepoIndexingStatus.FAILED).map((repo) => ({ - repoId: repo.id, - repoName: repo.name, - }))} - disabled={isDisabled} - /> - )) - ) : ( -
- -

No connections

-
- )} -
- ); -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/components/statusIcon.tsx b/packages/web/src/app/[domain]/connections/components/statusIcon.tsx deleted file mode 100644 index 4aad8eb6..00000000 --- a/packages/web/src/app/[domain]/connections/components/statusIcon.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import { cn } from "@/lib/utils"; -import { CircleCheckIcon, CircleXIcon } from "lucide-react"; -import { useMemo } from "react"; -import { FiLoader } from "react-icons/fi"; - -export type Status = 'waiting' | 'running' | 'succeeded' | 'succeeded-with-warnings' | 'garbage-collecting' | 'failed'; - -export const StatusIcon = ({ - status, - className, -}: { status: Status, className?: string }) => { - const Icon = useMemo(() => { - switch (status) { - case 'waiting': - case 'garbage-collecting': - case 'running': - return ; - case 'succeeded': - return ; - case 'failed': - return ; - case 'succeeded-with-warnings': - default: - return null; - } - }, [className, status]); - - return Icon; -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/layout.tsx b/packages/web/src/app/[domain]/connections/layout.tsx deleted file mode 100644 index 9aeb3541..00000000 --- a/packages/web/src/app/[domain]/connections/layout.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import { auth } from "@/auth"; -import { NavigationMenu } from "../components/navigationMenu"; -import { redirect } from "next/navigation"; - -interface LayoutProps { - children: React.ReactNode; - params: Promise<{ domain: string }>; -} - -export default async function Layout( - props: LayoutProps -) { - const params = await props.params; - - const { - domain - } = params; - - const { - children - } = props; - - const session = await auth(); - if (!session) { - return redirect(`/${domain}`); - } - - return ( -
- -
-
{children}
-
-
- ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/page.tsx b/packages/web/src/app/[domain]/connections/page.tsx deleted file mode 100644 index fdf3f32c..00000000 --- a/packages/web/src/app/[domain]/connections/page.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import { ConnectionList } from "./components/connectionList"; -import { Header } from "../components/header"; -import { getConnections, getOrgMembership } from "@/actions"; -import { isServiceError } from "@/lib/utils"; -import { notFound, ServiceErrorException } from "@/lib/serviceError"; -import { OrgRole } from "@sourcebot/db"; - -export default async function ConnectionsPage(props: { params: Promise<{ domain: string }> }) { - const params = await props.params; - - const { - domain - } = params; - - const connections = await getConnections(domain); - if (isServiceError(connections)) { - throw new ServiceErrorException(connections); - } - - const membership = await getOrgMembership(domain); - if (isServiceError(membership)) { - throw new ServiceErrorException(notFound()); - } - - return ( -
-
-

Connections

-
- -
- ); -} diff --git a/packages/web/src/app/[domain]/connections/quickActions.tsx b/packages/web/src/app/[domain]/connections/quickActions.tsx deleted file mode 100644 index 5f8008b0..00000000 --- a/packages/web/src/app/[domain]/connections/quickActions.tsx +++ /dev/null @@ -1,575 +0,0 @@ -import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type" -import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; -import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/bitbucket.type"; -import { QuickAction } from "../components/configEditor"; -import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; -import { GerritConnectionConfig } from "@sourcebot/schemas/v3/gerrit.type"; -import { CodeSnippet } from "@/app/components/codeSnippet"; - -export const githubQuickActions: QuickAction[] = [ - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - repos: [ - ...(previous.repos ?? []), - "/" - ] - }), - name: "Add a single repo", - selectionText: "/", - description: ( -
- Add a individual repository to sync with. Ensure the repository is visible to the provided token (if any). - Examples: -
- {[ - "sourcebot/sourcebot", - "vercel/next.js", - "torvalds/linux" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - orgs: [ - ...(previous.orgs ?? []), - "" - ] - }), - name: "Add an organization", - selectionText: "", - description: ( -
- Add an organization to sync with. All repositories in the organization visible to the provided token (if any) will be synced. - Examples: -
- {[ - "commaai", - "sourcebot", - "vercel" - ].map((org) => ( - {org} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - users: [ - ...(previous.users ?? []), - "" - ] - }), - name: "Add a user", - selectionText: "", - description: ( -
- Add a user to sync with. All repositories that the user owns visible to the provided token (if any) will be synced. - Examples: -
- {[ - "jane-doe", - "torvalds", - "octocat" - ].map((org) => ( - {org} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - url: previous.url ?? "https://github.example.com", - }), - name: "Set url to GitHub instance", - selectionText: "https://github.example.com", - description: Set a custom GitHub host. Defaults to https://github.com. - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - repos: [ - ...(previous.exclude?.repos ?? []), - "" - ] - } - }), - name: "Exclude by repo name", - selectionText: "", - description: ( -
- Exclude repositories from syncing by name. Glob patterns are supported. - Examples: -
- {[ - "my-org/docs*", - "my-org/test*" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - topics: [ - ...(previous.exclude?.topics ?? []), - "" - ] - } - }), - name: "Exclude by topic", - selectionText: "", - description: ( -
- Exclude topics from syncing. Only repos that do not match any of the provided topics will be synced. Glob patterns are supported. - Examples: -
- {[ - "docs", - "ci" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - topics: [ - ...(previous.topics ?? []), - "" - ] - }), - name: "Include by topic", - selectionText: "", - description: ( -
- Include repositories by topic. Only repos that match at least one of the provided topics will be synced. Glob patterns are supported. - Examples: -
- {[ - "docs", - "ci" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - archived: true, - } - }), - name: "Exclude archived repos", - description: Exclude archived repositories from syncing. - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - forks: true, - } - }), - name: "Exclude forked repos", - description: Exclude forked repositories from syncing. - } -]; - -export const gitlabQuickActions: QuickAction[] = [ - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - projects: [ - ...previous.projects ?? [], - "" - ] - }), - name: "Add a project", - selectionText: "", - description: ( -
- Add a individual project to sync with. Ensure the project is visible to the provided token (if any). - Examples: -
- {[ - "gitlab-org/gitlab", - "corp/team-project", - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - users: [ - ...previous.users ?? [], - "" - ] - }), - name: "Add a user", - selectionText: "", - description: ( -
- Add a user to sync with. All projects that the user owns visible to the provided token (if any) will be synced. - Examples: -
- {[ - "jane-doe", - "torvalds" - ].map((org) => ( - {org} - ))} -
-
- ) - }, - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - groups: [ - ...previous.groups ?? [], - "" - ] - }), - name: "Add a group", - selectionText: "", - description: ( -
- Add a group to sync with. All projects in the group (and recursive subgroups) visible to the provided token (if any) will be synced. - Examples: -
- {[ - "my-group", - "path/to/subgroup" - ].map((org) => ( - {org} - ))} -
-
- ) - }, - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - url: previous.url ?? "https://gitlab.example.com", - }), - name: "Set url to GitLab instance", - selectionText: "https://gitlab.example.com", - description: Set a custom GitLab host. Defaults to https://gitlab.com. - }, - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - all: true, - }), - name: "Sync all projects", - description: Sync all projects visible to the provided token (if any). Only available when using a self-hosted GitLab instance. - }, - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - projects: [ - ...(previous.exclude?.projects ?? []), - "" - ] - } - }), - name: "Exclude a project", - selectionText: "", - description: ( -
- List of projects to exclude from syncing. Glob patterns are supported. - Examples: -
- {[ - "docs/**", - "**/tests/**", - ].map((repo) => ( - {repo} - ))} -
-
- ) - } -] - -export const giteaQuickActions: QuickAction[] = [ - { - fn: (previous: GiteaConnectionConfig) => ({ - ...previous, - orgs: [ - ...(previous.orgs ?? []), - "" - ] - }), - name: "Add an organization", - selectionText: "", - }, - { - fn: (previous: GiteaConnectionConfig) => ({ - ...previous, - repos: [ - ...(previous.repos ?? []), - "/" - ] - }), - name: "Add a repo", - selectionText: "/", - }, - { - fn: (previous: GiteaConnectionConfig) => ({ - ...previous, - url: previous.url ?? "https://gitea.example.com", - }), - name: "Set url to Gitea instance", - selectionText: "https://gitea.example.com", - } -] - -export const gerritQuickActions: QuickAction[] = [ - { - fn: (previous: GerritConnectionConfig) => ({ - ...previous, - projects: [ - ...(previous.projects ?? []), - "" - ] - }), - name: "Add a project", - }, - { - fn: (previous: GerritConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - projects: [ - ...(previous.exclude?.projects ?? []), - "" - ] - } - }), - name: "Exclude a project", - } -] - -export const bitbucketCloudQuickActions: QuickAction[] = [ - { - // add user - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - user: previous.user ?? "username" - }), - name: "Add username", - selectionText: "username", - description: ( -
- Username to use for authentication. This is only required if you're using an App Password (stored in token) for authentication. -
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - workspaces: [ - ...(previous.workspaces ?? []), - "myWorkspace" - ] - }), - name: "Add a workspace", - selectionText: "myWorkspace", - description: ( -
- Add a workspace to sync with. Ensure the workspace is visible to the provided token (if any). -
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - repos: [ - ...(previous.repos ?? []), - "myWorkspace/myRepo" - ] - }), - name: "Add a repo", - selectionText: "myWorkspace/myRepo", - description: ( -
- Add an individual repository to sync with. Ensure the repository is visible to the provided token (if any). -
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - projects: [ - ...(previous.projects ?? []), - "myProject" - ] - }), - name: "Add a project", - selectionText: "myProject", - description: ( -
- Add a project to sync with. Ensure the project is visible to the provided token (if any). -
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - repos: [...(previous.exclude?.repos ?? []), "myWorkspace/myExcludedRepo"] - } - }), - name: "Exclude a repo", - selectionText: "myWorkspace/myExcludedRepo", - description: ( -
- Exclude a repository from syncing. Glob patterns are supported. -
- ) - }, - // exclude forked - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - forks: true - } - }), - name: "Exclude forked repos", - description: Exclude forked repositories from syncing. - } -] - -export const bitbucketDataCenterQuickActions: QuickAction[] = [ - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - url: previous.url ?? "https://bitbucket.example.com", - }), - name: "Set url to Bitbucket DC instance", - selectionText: "https://bitbucket.example.com", - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - repos: [ - ...(previous.repos ?? []), - "myProject/myRepo" - ] - }), - name: "Add a repo", - selectionText: "myProject/myRepo", - description: ( -
- Add a individual repository to sync with. Ensure the repository is visible to the provided token (if any). - Examples: -
- {[ - "PROJ/repo-name", - "MYPROJ/api" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - projects: [ - ...(previous.projects ?? []), - "myProject" - ] - }), - name: "Add a project", - selectionText: "myProject", - description: ( -
- Add a project to sync with. Ensure the project is visible to the provided token (if any). -
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - repos: [...(previous.exclude?.repos ?? []), "myProject/myExcludedRepo"] - } - }), - name: "Exclude a repo", - selectionText: "myProject/myExcludedRepo", - description: ( -
- Exclude a repository from syncing. Glob patterns are supported. - Examples: -
- {[ - "myProject/myExcludedRepo", - "myProject2/*" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - // exclude archived - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - archived: true - } - }), - name: "Exclude archived repos", - }, - // exclude forked - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - forks: true - } - }), - name: "Exclude forked repos", - } -] - diff --git a/packages/web/src/app/[domain]/connections/utils.ts b/packages/web/src/app/[domain]/connections/utils.ts deleted file mode 100644 index cd728092..00000000 --- a/packages/web/src/app/[domain]/connections/utils.ts +++ /dev/null @@ -1,40 +0,0 @@ -import Ajv, { Schema } from "ajv"; -import { z } from "zod"; - -export const createZodConnectionConfigValidator = (jsonSchema: Schema, additionalConfigValidation?: (config: T) => { message: string, isValid: boolean }) => { - const ajv = new Ajv({ - validateFormats: false, - }); - const validate = ajv.compile(jsonSchema); - - return z - .string() - .superRefine((data, ctx) => { - const addIssue = (message: string) => { - return ctx.addIssue({ - code: "custom", - message: `Schema validation error: ${message}` - }); - } - - let parsed; - try { - parsed = JSON.parse(data); - } catch { - addIssue("Invalid JSON"); - return; - } - - const valid = validate(parsed); - if (!valid) { - addIssue(ajv.errorsText(validate.errors)); - } - - if (additionalConfigValidation) { - const result = additionalConfigValidation(parsed as T); - if (!result.isValid) { - addIssue(result.message); - } - } - }); -} \ No newline at end of file From c56433d8a92cd57c04c229a06dd736b131855cc2 Mon Sep 17 00:00:00 2001 From: bkellam Date: Thu, 16 Oct 2025 23:12:41 -0700 Subject: [PATCH 12/24] repository table --- .../navigationMenu/navigationItems.tsx | 1 - .../navigationMenu/progressIndicator.tsx | 10 +- .../web/src/app/[domain]/repos/columns.tsx | 67 +++----- packages/web/src/app/[domain]/repos/page.tsx | 51 ++++-- .../app/[domain]/repos/repositoryTable.tsx | 160 ++++++++---------- .../web/src/app/[domain]/settings/layout.tsx | 4 - packages/web/src/components/ui/data-table.tsx | 2 +- packages/web/src/initialize.ts | 15 -- packages/web/src/prisma.ts | 4 + packages/web/src/withAuthV2.ts | 4 +- 10 files changed, 144 insertions(+), 174 deletions(-) diff --git a/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx b/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx index 055c8b10..9bcae00a 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx @@ -2,7 +2,6 @@ import { NavigationMenuItem, NavigationMenuLink, NavigationMenuList, navigationMenuTriggerStyle } from "@/components/ui/navigation-menu"; import { Badge } from "@/components/ui/badge"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { cn, getShortenedNumberDisplayString } from "@/lib/utils"; import { SearchIcon, MessageCircleIcon, BookMarkedIcon, SettingsIcon, CircleIcon } from "lucide-react"; import { usePathname } from "next/navigation"; diff --git a/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx b/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx index cc5d5478..8e1889df 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx @@ -1,5 +1,6 @@ 'use client'; +import { useToast } from "@/components/hooks/use-toast"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Separator } from "@/components/ui/separator"; @@ -25,6 +26,7 @@ export const ProgressIndicator = ({ }: ProgressIndicatorProps) => { const domain = useDomain(); const router = useRouter(); + const { toast } = useToast(); if (numRepos === 0) { return null; @@ -51,6 +53,9 @@ export const ProgressIndicator = ({ className="h-6 w-6 text-muted-foreground" onClick={() => { router.refresh(); + toast({ + description: "Page refreshed", + }); }} > @@ -105,14 +110,13 @@ const RepoItem = ({ repo }: { repo: RepositoryQuery }) => { return ( - {repoIcon} {displayName} - +
) } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/repos/columns.tsx b/packages/web/src/app/[domain]/repos/columns.tsx index 5ad1657e..a1cd1bdc 100644 --- a/packages/web/src/app/[domain]/repos/columns.tsx +++ b/packages/web/src/app/[domain]/repos/columns.tsx @@ -2,72 +2,51 @@ import { Button } from "@/components/ui/button" import type { ColumnDef } from "@tanstack/react-table" -import { ArrowUpDown, Clock, Loader2, CheckCircle2, XCircle, Trash2, Check, ListFilter } from "lucide-react" +import { ArrowUpDown, Clock, Loader2, CheckCircle2, Check, ListFilter } from "lucide-react" import Image from "next/image" import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" import { cn, getRepoImageSrc } from "@/lib/utils" -import { RepoIndexingStatus } from "@sourcebot/db"; import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu" import Link from "next/link" import { getBrowsePath } from "../browse/hooks/utils" +export type RepoStatus = 'syncing' | 'indexed' | 'not-indexed'; + export type RepositoryColumnInfo = { repoId: number repoName: string; repoDisplayName: string imageUrl?: string - repoIndexingStatus: RepoIndexingStatus + status: RepoStatus lastIndexed: string } -const statusLabels = { - [RepoIndexingStatus.NEW]: "Queued", - [RepoIndexingStatus.IN_INDEX_QUEUE]: "Queued", - [RepoIndexingStatus.INDEXING]: "Indexing", - [RepoIndexingStatus.INDEXED]: "Indexed", - [RepoIndexingStatus.FAILED]: "Failed", - [RepoIndexingStatus.IN_GC_QUEUE]: "Deleting", - [RepoIndexingStatus.GARBAGE_COLLECTING]: "Deleting", - [RepoIndexingStatus.GARBAGE_COLLECTION_FAILED]: "Deletion Failed" +const statusLabels: Record = { + 'syncing': "Syncing", + 'indexed': "Indexed", + 'not-indexed': "Pending", }; -const StatusIndicator = ({ status }: { status: RepoIndexingStatus }) => { +const StatusIndicator = ({ status }: { status: RepoStatus }) => { let icon = null let description = "" let className = "" switch (status) { - case RepoIndexingStatus.NEW: - case RepoIndexingStatus.IN_INDEX_QUEUE: - icon = - description = "Repository is queued for indexing" - className = "text-yellow-600 bg-yellow-50 dark:bg-yellow-900/20 dark:text-yellow-400" - break - case RepoIndexingStatus.INDEXING: + case 'syncing': icon = - description = "Repository is being indexed" + description = "Repository is currently syncing" className = "text-blue-600 bg-blue-50 dark:bg-blue-900/20 dark:text-blue-400" break - case RepoIndexingStatus.INDEXED: + case 'indexed': icon = - description = "Repository has been successfully indexed" + description = "Repository has been successfully indexed and is up to date" className = "text-green-600 bg-green-50 dark:bg-green-900/20 dark:text-green-400" break - case RepoIndexingStatus.FAILED: - icon = - description = "Repository indexing failed" - className = "text-red-600 bg-red-50 dark:bg-red-900/20 dark:text-red-400" - break - case RepoIndexingStatus.IN_GC_QUEUE: - case RepoIndexingStatus.GARBAGE_COLLECTING: - icon = - description = "Repository is being deleted" - className = "text-gray-600 bg-gray-50 dark:bg-gray-900/20 dark:text-gray-400" - break - case RepoIndexingStatus.GARBAGE_COLLECTION_FAILED: - icon = - description = "Repository deletion failed" - className = "text-red-600 bg-red-50 dark:bg-red-900/20 dark:text-red-400" + case 'not-indexed': + icon = + description = "Repository is pending initial sync" + className = "text-yellow-600 bg-yellow-50 dark:bg-yellow-900/20 dark:text-yellow-400" break } @@ -130,9 +109,9 @@ export const columns = (domain: string): ColumnDef[] => [ }, }, { - accessorKey: "repoIndexingStatus", + accessorKey: "status", header: ({ column }) => { - const uniqueLabels = Array.from(new Set(Object.values(statusLabels))); + const uniqueLabels = Object.values(statusLabels); const currentFilter = column.getFilterValue() as string | undefined; return ( @@ -173,12 +152,12 @@ export const columns = (domain: string): ColumnDef[] => [ ) }, cell: ({ row }) => { - return + return }, filterFn: (row, id, value) => { if (value === undefined) return true; - const status = row.getValue(id) as RepoIndexingStatus; + const status = row.getValue(id) as RepoStatus; return statusLabels[status] === value; }, }, @@ -191,14 +170,14 @@ export const columns = (domain: string): ColumnDef[] => [ onClick={() => column.toggleSorting(column.getIsSorted() === "asc")} className="px-0 font-medium hover:bg-transparent focus:bg-transparent active:bg-transparent focus-visible:ring-0 focus-visible:ring-offset-0" > - Last Indexed + Last Synced
), cell: ({ row }) => { if (!row.original.lastIndexed) { - return
-
; + return
Never
; } const date = new Date(row.original.lastIndexed) return ( diff --git a/packages/web/src/app/[domain]/repos/page.tsx b/packages/web/src/app/[domain]/repos/page.tsx index 4502dafc..60031811 100644 --- a/packages/web/src/app/[domain]/repos/page.tsx +++ b/packages/web/src/app/[domain]/repos/page.tsx @@ -1,8 +1,20 @@ -import { RepositoryTable } from "./repositoryTable"; -import { getOrgFromDomain } from "@/data/org"; -import { PageNotFound } from "../components/pageNotFound"; -import { Header } from "../components/header"; +import { auth } from "@/auth"; import { env } from "@/env.mjs"; +import { getPrismaClient } from "@/prisma"; +import { RepoJob } from "@sourcebot/db"; +import { Header } from "../components/header"; +import { RepoStatus } from "./columns"; +import { RepositoryTable } from "./repositoryTable"; + +function getRepoStatus(repo: { indexedAt: Date | null, jobs: RepoJob[] }): RepoStatus { + const latestJob = repo.jobs[0]; + + if (latestJob?.status === 'PENDING' || latestJob?.status === 'IN_PROGRESS') { + return 'syncing'; + } + + return repo.indexedAt ? 'indexed' : 'not-indexed'; +} export default async function ReposPage(props: { params: Promise<{ domain: string }> }) { const params = await props.params; @@ -11,22 +23,33 @@ export default async function ReposPage(props: { params: Promise<{ domain: strin domain } = params; - const org = await getOrgFromDomain(domain); - if (!org) { - return - } + const session = await auth(); + const prisma = getPrismaClient(session?.user?.id); + + const repos = await prisma.repo.findMany({ + include: { + jobs: true, + } + }); return (

Repositories

-
-
- -
+
+ ({ + repoId: repo.id, + repoName: repo.name, + repoDisplayName: repo.displayName ?? repo.name, + imageUrl: repo.imageUrl ?? undefined, + indexedAt: repo.indexedAt ?? undefined, + status: getRepoStatus(repo), + }))} + domain={domain} + isAddReposButtonVisible={env.EXPERIMENT_SELF_SERVE_REPO_INDEXING_ENABLED === 'true'} + />
) diff --git a/packages/web/src/app/[domain]/repos/repositoryTable.tsx b/packages/web/src/app/[domain]/repos/repositoryTable.tsx index 8d9dc0f1..9b67cca7 100644 --- a/packages/web/src/app/[domain]/repos/repositoryTable.tsx +++ b/packages/web/src/app/[domain]/repos/repositoryTable.tsx @@ -1,118 +1,81 @@ "use client"; -import { DataTable } from "@/components/ui/data-table"; -import { columns, RepositoryColumnInfo } from "./columns"; -import { unwrapServiceError } from "@/lib/utils"; -import { useQuery } from "@tanstack/react-query"; -import { useDomain } from "@/hooks/useDomain"; -import { RepoIndexingStatus } from "@sourcebot/db"; -import { useMemo } from "react"; -import { Skeleton } from "@/components/ui/skeleton"; -import { env } from "@/env.mjs"; +import { useToast } from "@/components/hooks/use-toast"; import { Button } from "@/components/ui/button"; -import { PlusIcon } from "lucide-react"; +import { DataTable } from "@/components/ui/data-table"; +import { PlusIcon, RefreshCwIcon } from "lucide-react"; +import { useRouter } from "next/navigation"; +import { useMemo, useState } from "react"; +import { columns, RepositoryColumnInfo, RepoStatus } from "./columns"; import { AddRepositoryDialog } from "./components/addRepositoryDialog"; -import { useState } from "react"; -import { getRepos } from "@/app/api/(client)/client"; interface RepositoryTableProps { - isAddReposButtonVisible: boolean + repos: { + repoId: number; + repoName: string; + repoDisplayName: string; + imageUrl?: string; + indexedAt?: Date; + status: RepoStatus; + }[]; + domain: string; + isAddReposButtonVisible: boolean; } export const RepositoryTable = ({ + repos, + domain, isAddReposButtonVisible, }: RepositoryTableProps) => { - const domain = useDomain(); const [isAddDialogOpen, setIsAddDialogOpen] = useState(false); - - const { data: repos, isLoading: reposLoading, error: reposError } = useQuery({ - queryKey: ['repos'], - queryFn: async () => { - return await unwrapServiceError(getRepos()); - }, - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - refetchIntervalInBackground: true, - }); + const router = useRouter(); + const { toast } = useToast(); const tableRepos = useMemo(() => { - if (reposLoading) return Array(4).fill(null).map(() => ({ - repoId: 0, - repoName: "", - repoDisplayName: "", - repoIndexingStatus: RepoIndexingStatus.NEW, - lastIndexed: "", - imageUrl: "", - })); - - if (!repos) return []; return repos.map((repo): RepositoryColumnInfo => ({ repoId: repo.repoId, repoName: repo.repoName, repoDisplayName: repo.repoDisplayName ?? repo.repoName, imageUrl: repo.imageUrl, - repoIndexingStatus: repo.repoIndexingStatus as RepoIndexingStatus, + status: repo.status, lastIndexed: repo.indexedAt?.toISOString() ?? "", })).sort((a, b) => { - const getPriorityFromStatus = (status: RepoIndexingStatus) => { + const getPriorityFromStatus = (status: RepoStatus) => { switch (status) { - case RepoIndexingStatus.IN_INDEX_QUEUE: - case RepoIndexingStatus.INDEXING: - return 0 // Highest priority - currently indexing - case RepoIndexingStatus.FAILED: - return 1 // Second priority - failed repos need attention - case RepoIndexingStatus.INDEXED: + case 'syncing': + return 0 // Highest priority - currently syncing + case 'not-indexed': + return 1 // Second priority - not yet indexed + case 'indexed': return 2 // Third priority - successfully indexed default: - return 3 // Lowest priority - other statuses (NEW, etc.) + return 3 } } // Sort by priority first - const aPriority = getPriorityFromStatus(a.repoIndexingStatus); - const bPriority = getPriorityFromStatus(b.repoIndexingStatus); - + const aPriority = getPriorityFromStatus(a.status); + const bPriority = getPriorityFromStatus(b.status); + if (aPriority !== bPriority) { - return aPriority - bPriority; // Lower priority number = higher precedence + return aPriority - bPriority; } - + // If same priority, sort by last indexed date (most recent first) - return new Date(b.lastIndexed).getTime() - new Date(a.lastIndexed).getTime(); + if (a.lastIndexed && b.lastIndexed) { + return new Date(b.lastIndexed).getTime() - new Date(a.lastIndexed).getTime(); + } + + // Put items without dates at the end + if (!a.lastIndexed) return 1; + if (!b.lastIndexed) return -1; + return 0; }); - }, [repos, reposLoading]); + }, [repos]); const tableColumns = useMemo(() => { - if (reposLoading) { - return columns(domain).map((column) => { - if ('accessorKey' in column && column.accessorKey === "name") { - return { - ...column, - cell: () => ( -
- {/* Avatar skeleton */} - {/* Repository name skeleton */} -
- ), - } - } - - return { - ...column, - cell: () => ( -
- -
- ), - } - }) - } - return columns(domain); - }, [reposLoading, domain]); - - - if (reposError) { - return
Error loading repositories
; - } + }, [domain]); return ( <> @@ -121,18 +84,35 @@ export const RepositoryTable = ({ data={tableRepos} searchKey="repoDisplayName" searchPlaceholder="Search repositories..." - headerActions={isAddReposButtonVisible && ( - + headerActions={( +
+ + {isAddReposButtonVisible && ( + + )} +
)} /> - + ({ return (
-
+
repo.repo.repoIndexingStatus === RepoIndexingStatus.FAILED).map(repo => repo.repo.id) ?? []; - if (failedRepos.length > 0) { - await prisma.repo.updateMany({ - where: { - id: { - in: failedRepos, - } - }, - data: { - repoIndexingStatus: RepoIndexingStatus.NEW, - } - }) - } } } diff --git a/packages/web/src/prisma.ts b/packages/web/src/prisma.ts index 1d4b7585..6bd9aff8 100644 --- a/packages/web/src/prisma.ts +++ b/packages/web/src/prisma.ts @@ -57,4 +57,8 @@ export const userScopedPrismaClientExtension = (userId?: string) => { } }) }) +} + +export const getPrismaClient = (userId?: string) => { + return prisma.$extends(userScopedPrismaClientExtension(userId)) as PrismaClient; } \ No newline at end of file diff --git a/packages/web/src/withAuthV2.ts b/packages/web/src/withAuthV2.ts index c6cbb8bb..1ffd541e 100644 --- a/packages/web/src/withAuthV2.ts +++ b/packages/web/src/withAuthV2.ts @@ -1,4 +1,4 @@ -import { prisma as __unsafePrisma, userScopedPrismaClientExtension } from "@/prisma"; +import { getPrismaClient, prisma as __unsafePrisma } from "@/prisma"; import { hashSecret } from "@sourcebot/crypto"; import { ApiKey, Org, OrgRole, PrismaClient, User } from "@sourcebot/db"; import { headers } from "next/headers"; @@ -88,7 +88,7 @@ export const getAuthContext = async (): Promise Date: Fri, 17 Oct 2025 14:39:35 -0700 Subject: [PATCH 13/24] remove dead code --- packages/web/src/actions.ts | 477 +----------------- .../navigationMenu/navigationItems.tsx | 4 +- packages/web/src/lib/constants.ts | 1 - 3 files changed, 16 insertions(+), 466 deletions(-) diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 3bfcee35..d191a2ca 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -5,46 +5,32 @@ import { env } from "@/env.mjs"; import { addUserToOrganization, orgHasAvailability } from "@/lib/authUtils"; import { ErrorCode } from "@/lib/errorCodes"; import { notAuthenticated, notFound, orgNotFound, secretAlreadyExists, ServiceError, ServiceErrorException, unexpectedError } from "@/lib/serviceError"; -import { CodeHostType, getOrgMetadata, isHttpError, isServiceError } from "@/lib/utils"; +import { getOrgMetadata, isHttpError, isServiceError } from "@/lib/utils"; import { prisma } from "@/prisma"; import { render } from "@react-email/components"; import * as Sentry from '@sentry/nextjs'; -import { decrypt, encrypt, generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto"; -import { ApiKey, ConnectionSyncStatus, Org, OrgRole, Prisma, RepoIndexingStatus, RepoJobStatus, RepoJobType, StripeSubscriptionStatus } from "@sourcebot/db"; +import { encrypt, generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto"; +import { ApiKey, Org, OrgRole, Prisma, RepoJobStatus, RepoJobType, StripeSubscriptionStatus } from "@sourcebot/db"; import { createLogger } from "@sourcebot/logger"; -import { azuredevopsSchema } from "@sourcebot/schemas/v3/azuredevops.schema"; -import { bitbucketSchema } from "@sourcebot/schemas/v3/bitbucket.schema"; -import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; -import { genericGitHostSchema } from "@sourcebot/schemas/v3/genericGitHost.schema"; -import { gerritSchema } from "@sourcebot/schemas/v3/gerrit.schema"; -import { giteaSchema } from "@sourcebot/schemas/v3/gitea.schema"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; -import { githubSchema } from "@sourcebot/schemas/v3/github.schema"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; -import { gitlabSchema } from "@sourcebot/schemas/v3/gitlab.schema"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; import { getPlan, hasEntitlement } from "@sourcebot/shared"; -import Ajv from "ajv"; import { StatusCodes } from "http-status-codes"; import { cookies, headers } from "next/headers"; import { createTransport } from "nodemailer"; import { Octokit } from "octokit"; import { auth } from "./auth"; -import { getConnection } from "./data/connection"; import { getOrgFromDomain } from "./data/org"; import { decrementOrgSeatCount, getSubscriptionForOrg } from "./ee/features/billing/serverUtils"; import { IS_BILLING_ENABLED } from "./ee/features/billing/stripe"; import InviteUserEmail from "./emails/inviteUserEmail"; import JoinRequestApprovedEmail from "./emails/joinRequestApprovedEmail"; import JoinRequestSubmittedEmail from "./emails/joinRequestSubmittedEmail"; -import { AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME, MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, SEARCH_MODE_COOKIE_NAME, SINGLE_TENANT_ORG_DOMAIN, SOURCEBOT_GUEST_USER_ID, SOURCEBOT_SUPPORT_EMAIL } from "./lib/constants"; +import { AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME, MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, SINGLE_TENANT_ORG_DOMAIN, SOURCEBOT_GUEST_USER_ID, SOURCEBOT_SUPPORT_EMAIL } from "./lib/constants"; import { orgDomainSchema, orgNameSchema, repositoryQuerySchema } from "./lib/schemas"; import { ApiKeyPayload, TenancyMode } from "./lib/types"; -import { withAuthV2, withOptionalAuthV2 } from "./withAuthV2"; - -const ajv = new Ajv({ - validateFormats: false, -}); +import { withOptionalAuthV2 } from "./withAuthV2"; const logger = createLogger('web-actions'); const auditService = getAuditService(); @@ -187,31 +173,6 @@ export const withTenancyModeEnforcement = async(mode: TenancyMode, fn: () => ////// Actions /////// -export const createOrg = async (name: string, domain: string): Promise<{ id: number } | ServiceError> => sew(() => - withTenancyModeEnforcement('multi', () => - withAuth(async (userId) => { - const org = await prisma.org.create({ - data: { - name, - domain, - members: { - create: { - role: "OWNER", - user: { - connect: { - id: userId, - } - } - } - } - } - }); - - return { - id: org.id, - } - }))); - export const updateOrgName = async (name: string, domain: string) => sew(() => withAuth((userId) => withOrgMembership(userId, domain, async ({ org }) => { @@ -573,71 +534,6 @@ export const getUserApiKeys = async (domain: string): Promise<{ name: string; cr })); }))); -export const getConnections = async (domain: string, filter: { status?: ConnectionSyncStatus[] } = {}) => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connections = await prisma.connection.findMany({ - where: { - orgId: org.id, - ...(filter.status ? { - syncStatus: { in: filter.status } - } : {}), - }, - include: { - repos: { - include: { - repo: true, - } - } - } - }); - - return connections.map((connection) => ({ - id: connection.id, - name: connection.name, - syncStatus: connection.syncStatus, - syncStatusMetadata: connection.syncStatusMetadata, - connectionType: connection.connectionType, - updatedAt: connection.updatedAt, - syncedAt: connection.syncedAt ?? undefined, - linkedRepos: connection.repos.map(({ repo }) => ({ - id: repo.id, - name: repo.name, - repoIndexingStatus: repo.repoIndexingStatus, - })), - })); - }) - )); - -export const getConnectionInfo = async (connectionId: number, domain: string) => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connection = await prisma.connection.findUnique({ - where: { - id: connectionId, - orgId: org.id, - }, - include: { - repos: true, - } - }); - - if (!connection) { - return notFound(); - } - - return { - id: connection.id, - name: connection.name, - syncStatus: connection.syncStatus, - syncStatusMetadata: connection.syncStatusMetadata, - connectionType: connection.connectionType, - updatedAt: connection.updatedAt, - syncedAt: connection.syncedAt ?? undefined, - numLinkedRepos: connection.repos.length, - } - }))); - export const getRepos = async ({ where, take, @@ -781,54 +677,6 @@ export const getRepoInfoByName = async (repoName: string) => sew(() => } })); -export const createConnection = async (name: string, type: CodeHostType, connectionConfig: string, domain: string): Promise<{ id: number } | ServiceError> => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - if (env.CONFIG_PATH !== undefined) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.CONNECTION_CONFIG_PATH_SET, - message: "A configuration file has been provided. New connections cannot be added through the web interface.", - } satisfies ServiceError; - } - - const parsedConfig = parseConnectionConfig(connectionConfig); - if (isServiceError(parsedConfig)) { - return parsedConfig; - } - - const existingConnectionWithName = await prisma.connection.findUnique({ - where: { - name_orgId: { - orgId: org.id, - name, - } - } - }); - - if (existingConnectionWithName) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.CONNECTION_ALREADY_EXISTS, - message: "A connection with this name already exists.", - } satisfies ServiceError; - } - - const connection = await prisma.connection.create({ - data: { - orgId: org.id, - name, - config: parsedConfig as unknown as Prisma.InputJsonValue, - connectionType: type, - } - }); - - return { - id: connection.id, - } - }, OrgRole.OWNER) - )); - export const experimental_addGithubRepositoryByUrl = async (repositoryUrl: string): Promise<{ connectionId: number } | ServiceError> => sew(() => withOptionalAuthV2(async ({ org, prisma }) => { if (env.EXPERIMENT_SELF_SERVE_REPO_INDEXING_ENABLED !== 'true') { @@ -965,148 +813,6 @@ export const experimental_addGithubRepositoryByUrl = async (repositoryUrl: strin } })); -export const updateConnectionDisplayName = async (connectionId: number, name: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connection = await getConnection(connectionId, org.id); - if (!connection) { - return notFound(); - } - - const existingConnectionWithName = await prisma.connection.findUnique({ - where: { - name_orgId: { - orgId: org.id, - name, - } - } - }); - - if (existingConnectionWithName) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.CONNECTION_ALREADY_EXISTS, - message: "A connection with this name already exists.", - } satisfies ServiceError; - } - - await prisma.connection.update({ - where: { - id: connectionId, - orgId: org.id, - }, - data: { - name, - } - }); - - return { - success: true, - } - }, OrgRole.OWNER) - )); - -export const updateConnectionConfigAndScheduleSync = async (connectionId: number, config: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connection = await getConnection(connectionId, org.id); - if (!connection) { - return notFound(); - } - - const parsedConfig = parseConnectionConfig(config); - if (isServiceError(parsedConfig)) { - return parsedConfig; - } - - if (connection.syncStatus === "SYNC_NEEDED" || - connection.syncStatus === "IN_SYNC_QUEUE" || - connection.syncStatus === "SYNCING") { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.CONNECTION_SYNC_ALREADY_SCHEDULED, - message: "Connection is already syncing. Please wait for the sync to complete before updating the connection.", - } satisfies ServiceError; - } - - await prisma.connection.update({ - where: { - id: connectionId, - orgId: org.id, - }, - data: { - config: parsedConfig as unknown as Prisma.InputJsonValue, - syncStatus: "SYNC_NEEDED", - } - }); - - return { - success: true, - } - }, OrgRole.OWNER) - )); - -export const flagConnectionForSync = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connection = await getConnection(connectionId, org.id); - if (!connection || connection.orgId !== org.id) { - return notFound(); - } - - await prisma.connection.update({ - where: { - id: connection.id, - }, - data: { - syncStatus: "SYNC_NEEDED", - } - }); - - return { - success: true, - } - }) - )); - -export const flagReposForIndex = async (repoIds: number[]) => sew(() => - withAuthV2(async ({ org, prisma }) => { - await prisma.repo.updateMany({ - where: { - id: { in: repoIds }, - orgId: org.id, - }, - data: { - repoIndexingStatus: RepoIndexingStatus.NEW, - } - }); - - return { - success: true, - } - })); - -export const deleteConnection = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connection = await getConnection(connectionId, org.id); - if (!connection) { - return notFound(); - } - - await prisma.connection.delete({ - where: { - id: connectionId, - orgId: org.id, - } - }); - - return { - success: true, - } - }, OrgRole.OWNER) - )); - export const getCurrentUserRole = async (domain: string): Promise => sew(() => withAuth((userId) => withOrgMembership(userId, domain, async ({ userRole }) => { @@ -1319,13 +1025,6 @@ export const cancelInvite = async (inviteId: string, domain: string): Promise<{ }, /* minRequiredRole = */ OrgRole.OWNER) )); -export const getOrgInviteId = async (domain: string) => sew(() => - withAuth(async (userId) => - withOrgMembership(userId, domain, async ({ org }) => { - return org.inviteLinkId; - }, /* minRequiredRole = */ OrgRole.OWNER) - )); - export const getMe = async () => sew(() => withAuth(async (userId) => { const user = await prisma.user.findUnique({ @@ -1662,27 +1361,6 @@ export const leaveOrg = async (domain: string): Promise<{ success: boolean } | S }) )); - -export const getOrgMembership = async (domain: string) => sew(() => - withAuth(async (userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const membership = await prisma.userToOrg.findUnique({ - where: { - orgId_userId: { - orgId: org.id, - userId: userId, - } - } - }); - - if (!membership) { - return notFound(); - } - - return membership; - }) - )); - export const getOrgMembers = async (domain: string) => sew(() => withAuth(async (userId) => withOrgMembership(userId, domain, async ({ org }) => { @@ -1878,20 +1556,6 @@ export const setMemberApprovalRequired = async (domain: string, required: boolea ) ); -export const getInviteLinkEnabled = async (domain: string): Promise => sew(async () => { - const org = await prisma.org.findUnique({ - where: { - domain, - }, - }); - - if (!org) { - return orgNotFound(); - } - - return org.inviteLinkEnabled; -}); - export const setInviteLinkEnabled = async (domain: string, enabled: boolean): Promise<{ success: boolean } | ServiceError> => sew(async () => withAuth(async (userId) => withOrgMembership(userId, domain, async ({ org }) => { @@ -2023,10 +1687,6 @@ export const rejectAccountRequest = async (requestId: string, domain: string) => }, /* minRequiredRole = */ OrgRole.OWNER) )); -export const dismissMobileUnsupportedSplashScreen = async () => sew(async () => { - await (await cookies()).set(MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, 'true'); - return true; -}); export const getSearchContexts = async (domain: string) => sew(() => withAuth((userId) => @@ -2179,126 +1839,17 @@ export const setAnonymousAccessStatus = async (domain: string, enabled: boolean) }); }); -export async function setSearchModeCookie(searchMode: "precise" | "agentic") { - const cookieStore = await cookies(); - cookieStore.set(SEARCH_MODE_COOKIE_NAME, searchMode, { - httpOnly: false, // Allow client-side access - }); -} - -export async function setAgenticSearchTutorialDismissedCookie(dismissed: boolean) { +export const setAgenticSearchTutorialDismissedCookie = async (dismissed: boolean) => sew(async () => { const cookieStore = await cookies(); cookieStore.set(AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME, dismissed ? "true" : "false", { httpOnly: false, // Allow client-side access + maxAge: 365 * 24 * 60 * 60, // 1 year in seconds }); -} - -////// Helpers /////// - -const parseConnectionConfig = (config: string) => { - let parsedConfig: ConnectionConfig; - try { - parsedConfig = JSON.parse(config); - } catch (_e) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: "config must be a valid JSON object." - } satisfies ServiceError; - } - - const connectionType = parsedConfig.type; - const schema = (() => { - switch (connectionType) { - case "github": - return githubSchema; - case "gitlab": - return gitlabSchema; - case 'gitea': - return giteaSchema; - case 'gerrit': - return gerritSchema; - case 'bitbucket': - return bitbucketSchema; - case 'azuredevops': - return azuredevopsSchema; - case 'git': - return genericGitHostSchema; - } - })(); - - if (!schema) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: "invalid connection type", - } satisfies ServiceError; - } - - const isValidConfig = ajv.validate(schema, parsedConfig); - if (!isValidConfig) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: `config schema validation failed with errors: ${ajv.errorsText(ajv.errors)}`, - } satisfies ServiceError; - } - - if ('token' in parsedConfig && parsedConfig.token && 'env' in parsedConfig.token) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: "Environment variables are not supported for connections created in the web UI. Please use a secret instead.", - } satisfies ServiceError; - } - - const { numRepos, hasToken } = (() => { - switch (connectionType) { - case "gitea": - case "github": - case "bitbucket": - case "azuredevops": { - return { - numRepos: parsedConfig.repos?.length, - hasToken: !!parsedConfig.token, - } - } - case "gitlab": { - return { - numRepos: parsedConfig.projects?.length, - hasToken: !!parsedConfig.token, - } - } - case "gerrit": { - return { - numRepos: parsedConfig.projects?.length, - hasToken: true, // gerrit doesn't use a token atm - } - } - case "git": { - return { - numRepos: 1, - hasToken: false, - } - } - } - })(); - - if (!hasToken && numRepos && numRepos > env.CONFIG_MAX_REPOS_NO_TOKEN) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: `You must provide a token to sync more than ${env.CONFIG_MAX_REPOS_NO_TOKEN} repositories.`, - } satisfies ServiceError; - } - - return parsedConfig; -} - -export const encryptValue = async (value: string) => { - return encrypt(value); -} + return true; +}); -export const decryptValue = async (iv: string, encryptedValue: string) => { - return decrypt(iv, encryptedValue); -} \ No newline at end of file +export const dismissMobileUnsupportedSplashScreen = async () => sew(async () => { + const cookieStore = await cookies(); + cookieStore.set(MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, 'true'); + return true; +}); diff --git a/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx b/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx index 9bcae00a..52db3315 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx @@ -32,13 +32,13 @@ export const NavigationItems = ({ Search - {isActive(`/${domain}`) && } + {((isActive(`/${domain}`) || isActive(`/${domain}/search`)) && )} Date: Fri, 17 Oct 2025 14:47:53 -0700 Subject: [PATCH 14/24] wrap repos query in withOptionalAuthV2 --- packages/web/src/app/[domain]/repos/page.tsx | 29 +++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/web/src/app/[domain]/repos/page.tsx b/packages/web/src/app/[domain]/repos/page.tsx index 60031811..c5ed9689 100644 --- a/packages/web/src/app/[domain]/repos/page.tsx +++ b/packages/web/src/app/[domain]/repos/page.tsx @@ -1,10 +1,12 @@ -import { auth } from "@/auth"; import { env } from "@/env.mjs"; -import { getPrismaClient } from "@/prisma"; import { RepoJob } from "@sourcebot/db"; import { Header } from "../components/header"; import { RepoStatus } from "./columns"; import { RepositoryTable } from "./repositoryTable"; +import { sew } from "@/actions"; +import { withOptionalAuthV2 } from "@/withAuthV2"; +import { isServiceError } from "@/lib/utils"; +import { ServiceErrorException } from "@/lib/serviceError"; function getRepoStatus(repo: { indexedAt: Date | null, jobs: RepoJob[] }): RepoStatus { const latestJob = repo.jobs[0]; @@ -23,14 +25,10 @@ export default async function ReposPage(props: { params: Promise<{ domain: strin domain } = params; - const session = await auth(); - const prisma = getPrismaClient(session?.user?.id); - - const repos = await prisma.repo.findMany({ - include: { - jobs: true, - } - }); + const repos = await getReposWithJobs(); + if (isServiceError(repos)) { + throw new ServiceErrorException(repos); + } return (
@@ -54,3 +52,14 @@ export default async function ReposPage(props: { params: Promise<{ domain: strin
) } + +const getReposWithJobs = async () => sew(() => + withOptionalAuthV2(async ({ prisma }) => { + const repos = await prisma.repo.findMany({ + include: { + jobs: true, + } + }); + + return repos; + })); \ No newline at end of file From 9f4fb8480a5cb6cfb7aae4e598a1fb75b5a03dac Mon Sep 17 00:00:00 2001 From: bkellam Date: Fri, 17 Oct 2025 14:48:10 -0700 Subject: [PATCH 15/24] Raise search result default count to 5000 --- .../src/app/[domain]/search/components/searchResultsPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/src/app/[domain]/search/components/searchResultsPage.tsx b/packages/web/src/app/[domain]/search/components/searchResultsPage.tsx index 7391c536..553ee132 100644 --- a/packages/web/src/app/[domain]/search/components/searchResultsPage.tsx +++ b/packages/web/src/app/[domain]/search/components/searchResultsPage.tsx @@ -35,7 +35,7 @@ import { FilterPanel } from "./filterPanel"; import { useFilteredMatches } from "./filterPanel/useFilterMatches"; import { SearchResultsPanel } from "./searchResultsPanel"; -const DEFAULT_MAX_MATCH_COUNT = 500; +const DEFAULT_MAX_MATCH_COUNT = 5000; interface SearchResultsPageProps { searchQuery: string; From 721b2420c4be5451793348e78f3e7fa512995e1d Mon Sep 17 00:00:00 2001 From: bkellam Date: Fri, 17 Oct 2025 16:31:44 -0700 Subject: [PATCH 16/24] remove references to repoIndexingStatus --- packages/mcp/src/schemas.ts | 12 ------------ packages/web/src/actions.ts | 2 -- packages/web/src/initialize.ts | 2 +- packages/web/src/lib/schemas.ts | 2 -- 4 files changed, 1 insertion(+), 17 deletions(-) diff --git a/packages/mcp/src/schemas.ts b/packages/mcp/src/schemas.ts index 0bb8ff9a..b477e8f1 100644 --- a/packages/mcp/src/schemas.ts +++ b/packages/mcp/src/schemas.ts @@ -143,17 +143,6 @@ export const searchResponseSchema = z.object({ isSearchExhaustive: z.boolean(), }); -enum RepoIndexingStatus { - NEW = 'NEW', - IN_INDEX_QUEUE = 'IN_INDEX_QUEUE', - INDEXING = 'INDEXING', - INDEXED = 'INDEXED', - FAILED = 'FAILED', - IN_GC_QUEUE = 'IN_GC_QUEUE', - GARBAGE_COLLECTING = 'GARBAGE_COLLECTING', - GARBAGE_COLLECTION_FAILED = 'GARBAGE_COLLECTION_FAILED' -} - export const repositoryQuerySchema = z.object({ codeHostType: z.string(), repoId: z.number(), @@ -163,7 +152,6 @@ export const repositoryQuerySchema = z.object({ webUrl: z.string().optional(), imageUrl: z.string().optional(), indexedAt: z.coerce.date().optional(), - repoIndexingStatus: z.nativeEnum(RepoIndexingStatus), }); export const listRepositoriesResponseSchema = repositoryQuerySchema.array(); diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index d191a2ca..4f605d8e 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -559,7 +559,6 @@ export const getRepos = async ({ webUrl: repo.webUrl ?? undefined, imageUrl: repo.imageUrl ?? undefined, indexedAt: repo.indexedAt ?? undefined, - repoIndexingStatus: repo.repoIndexingStatus, })) })); @@ -673,7 +672,6 @@ export const getRepoInfoByName = async (repoName: string) => sew(() => webUrl: repo.webUrl ?? undefined, imageUrl: repo.imageUrl ?? undefined, indexedAt: repo.indexedAt ?? undefined, - repoIndexingStatus: repo.repoIndexingStatus, } })); diff --git a/packages/web/src/initialize.ts b/packages/web/src/initialize.ts index c3302da5..1063e58c 100644 --- a/packages/web/src/initialize.ts +++ b/packages/web/src/initialize.ts @@ -1,4 +1,4 @@ -import { ConnectionSyncStatus, OrgRole, Prisma, RepoIndexingStatus } from '@sourcebot/db'; +import { ConnectionSyncStatus, OrgRole, Prisma } from '@sourcebot/db'; import { env } from './env.mjs'; import { prisma } from "@/prisma"; import { SINGLE_TENANT_ORG_ID, SINGLE_TENANT_ORG_DOMAIN, SOURCEBOT_GUEST_USER_ID, SINGLE_TENANT_ORG_NAME } from './lib/constants'; diff --git a/packages/web/src/lib/schemas.ts b/packages/web/src/lib/schemas.ts index 7d37d9ca..fa8fbc7a 100644 --- a/packages/web/src/lib/schemas.ts +++ b/packages/web/src/lib/schemas.ts @@ -1,5 +1,4 @@ import { checkIfOrgDomainExists } from "@/actions"; -import { RepoIndexingStatus } from "@sourcebot/db"; import { z } from "zod"; import { isServiceError } from "./utils"; import { serviceErrorSchema } from "./serviceError"; @@ -22,7 +21,6 @@ export const repositoryQuerySchema = z.object({ webUrl: z.string().optional(), imageUrl: z.string().optional(), indexedAt: z.coerce.date().optional(), - repoIndexingStatus: z.nativeEnum(RepoIndexingStatus), }); export const searchContextQuerySchema = z.object({ From 3f0947ceb79d3732fcc3575becc5818b02f75bda Mon Sep 17 00:00:00 2001 From: bkellam Date: Fri, 17 Oct 2025 16:36:07 -0700 Subject: [PATCH 17/24] Remove NEXT_PUBLIC_POLLING_INTERVAL_MS env var --- .env.development | 3 --- packages/web/src/env.mjs | 2 -- 2 files changed, 5 deletions(-) diff --git a/.env.development b/.env.development index 1740c2df..9b5b9682 100644 --- a/.env.development +++ b/.env.development @@ -80,9 +80,6 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection # Controls the number of concurrent indexing jobs that can run at once # INDEX_CONCURRENCY_MULTIPLE= -# Controls the polling interval for the web app -# NEXT_PUBLIC_POLLING_INTERVAL_MS= - # Controls the version of the web app # NEXT_PUBLIC_SOURCEBOT_VERSION= diff --git a/packages/web/src/env.mjs b/packages/web/src/env.mjs index 7a9c1589..f515c3d9 100644 --- a/packages/web/src/env.mjs +++ b/packages/web/src/env.mjs @@ -146,7 +146,6 @@ export const env = createEnv({ // Misc NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default('unknown'), - NEXT_PUBLIC_POLLING_INTERVAL_MS: numberSchema.default(5000), NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(), @@ -157,7 +156,6 @@ export const env = createEnv({ experimental__runtimeEnv: { NEXT_PUBLIC_POSTHOG_PAPIK: process.env.NEXT_PUBLIC_POSTHOG_PAPIK, NEXT_PUBLIC_SOURCEBOT_VERSION: process.env.NEXT_PUBLIC_SOURCEBOT_VERSION, - NEXT_PUBLIC_POLLING_INTERVAL_MS: process.env.NEXT_PUBLIC_POLLING_INTERVAL_MS, NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT, NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY, NEXT_PUBLIC_LANGFUSE_BASE_URL: process.env.NEXT_PUBLIC_LANGFUSE_BASE_URL, From 3f72a533c6c558ed45323a5812fc162ed8d38588 Mon Sep 17 00:00:00 2001 From: bkellam Date: Sat, 18 Oct 2025 14:03:05 -0700 Subject: [PATCH 18/24] fix tests --- packages/backend/vitest.config.ts | 3 +++ packages/web/src/prisma.ts | 4 ---- packages/web/src/withAuthV2.ts | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/backend/vitest.config.ts b/packages/backend/vitest.config.ts index 7c052526..5b2ab0d5 100644 --- a/packages/backend/vitest.config.ts +++ b/packages/backend/vitest.config.ts @@ -4,5 +4,8 @@ export default defineConfig({ test: { environment: 'node', watch: false, + env: { + DATA_CACHE_DIR: 'test-data' + } } }); \ No newline at end of file diff --git a/packages/web/src/prisma.ts b/packages/web/src/prisma.ts index 6bd9aff8..860d23fa 100644 --- a/packages/web/src/prisma.ts +++ b/packages/web/src/prisma.ts @@ -58,7 +58,3 @@ export const userScopedPrismaClientExtension = (userId?: string) => { }) }) } - -export const getPrismaClient = (userId?: string) => { - return prisma.$extends(userScopedPrismaClientExtension(userId)) as PrismaClient; -} \ No newline at end of file diff --git a/packages/web/src/withAuthV2.ts b/packages/web/src/withAuthV2.ts index 1ffd541e..c6cbb8bb 100644 --- a/packages/web/src/withAuthV2.ts +++ b/packages/web/src/withAuthV2.ts @@ -1,4 +1,4 @@ -import { getPrismaClient, prisma as __unsafePrisma } from "@/prisma"; +import { prisma as __unsafePrisma, userScopedPrismaClientExtension } from "@/prisma"; import { hashSecret } from "@sourcebot/crypto"; import { ApiKey, Org, OrgRole, PrismaClient, User } from "@sourcebot/db"; import { headers } from "next/headers"; @@ -88,7 +88,7 @@ export const getAuthContext = async (): Promise Date: Sat, 18 Oct 2025 14:30:46 -0700 Subject: [PATCH 19/24] db migration --- packages/backend/src/repoIndexManager.ts | 48 +++++++++---------- packages/backend/src/utils.ts | 2 +- .../migration.sql | 34 +++++++++++++ packages/db/prisma/schema.prisma | 26 +++------- packages/web/src/actions.ts | 8 ++-- .../components/navigationMenu/index.tsx | 8 ++-- packages/web/src/app/[domain]/repos/page.tsx | 4 +- 7 files changed, 75 insertions(+), 55 deletions(-) create mode 100644 packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts index e296220d..2f4e1031 100644 --- a/packages/backend/src/repoIndexManager.ts +++ b/packages/backend/src/repoIndexManager.ts @@ -1,5 +1,5 @@ import * as Sentry from '@sentry/node'; -import { PrismaClient, Repo, RepoJobStatus, RepoJobType } from "@sourcebot/db"; +import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import { createLogger, Logger } from "@sourcebot/logger"; import { existsSync } from 'fs'; import { readdir, rm } from 'fs/promises'; @@ -97,7 +97,7 @@ export class RepoIndexManager { some: { AND: [ { - type: RepoJobType.INDEX, + type: RepoIndexingJobType.INDEX, }, { OR: [ @@ -108,8 +108,8 @@ export class RepoIndexManager { { status: { in: [ - RepoJobStatus.PENDING, - RepoJobStatus.IN_PROGRESS, + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, ] }, }, @@ -123,7 +123,7 @@ export class RepoIndexManager { // Don't schedule if there are recent failed jobs (within the threshold date). { AND: [ - { status: RepoJobStatus.FAILED }, + { status: RepoIndexingJobStatus.FAILED }, { completedAt: { gt: thresholdDate } }, ] } @@ -139,7 +139,7 @@ export class RepoIndexManager { }); if (reposToIndex.length > 0) { - await this.createJobs(reposToIndex, RepoJobType.INDEX); + await this.createJobs(reposToIndex, RepoIndexingJobType.INDEX); } } @@ -161,13 +161,13 @@ export class RepoIndexManager { some: { AND: [ { - type: RepoJobType.CLEANUP, + type: RepoIndexingJobType.CLEANUP, }, { status: { in: [ - RepoJobStatus.PENDING, - RepoJobStatus.IN_PROGRESS, + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, ] }, }, @@ -184,15 +184,15 @@ export class RepoIndexManager { }); if (reposToCleanup.length > 0) { - await this.createJobs(reposToCleanup, RepoJobType.CLEANUP); + await this.createJobs(reposToCleanup, RepoIndexingJobType.CLEANUP); } } - private async createJobs(repos: Repo[], type: RepoJobType) { + private async createJobs(repos: Repo[], type: RepoIndexingJobType) { // @note: we don't perform this in a transaction because // we want to avoid the situation where a job is created and run // prior to the transaction being committed. - const jobs = await this.db.repoJob.createManyAndReturn({ + const jobs = await this.db.repoIndexingJob.createManyAndReturn({ data: repos.map(repo => ({ type, repoId: repo.id, @@ -222,12 +222,12 @@ export class RepoIndexManager { logger.info(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId}) (attempt ${job.attempts + 1} / ${job.maxAttempts})`); - const { repo, type: jobType } = await this.db.repoJob.update({ + const { repo, type: jobType } = await this.db.repoIndexingJob.update({ where: { id, }, data: { - status: RepoJobStatus.IN_PROGRESS, + status: RepoIndexingJobStatus.IN_PROGRESS, }, select: { type: true, @@ -253,9 +253,9 @@ export class RepoIndexManager { process.on('SIGINT', signalHandler); try { - if (jobType === RepoJobType.INDEX) { + if (jobType === RepoIndexingJobType.INDEX) { await this.indexRepository(repo, logger, abortController.signal); - } else if (jobType === RepoJobType.CLEANUP) { + } else if (jobType === RepoIndexingJobType.CLEANUP) { await this.cleanupRepository(repo, logger); } } finally { @@ -370,15 +370,15 @@ export class RepoIndexManager { private onJobCompleted = async (job: Job) => groupmqLifecycleExceptionWrapper('onJobCompleted', logger, async () => { const logger = createJobLogger(job.data.jobId); - const jobData = await this.db.repoJob.update({ + const jobData = await this.db.repoIndexingJob.update({ where: { id: job.data.jobId }, data: { - status: RepoJobStatus.COMPLETED, + status: RepoIndexingJobStatus.COMPLETED, completedAt: new Date(), } }); - if (jobData.type === RepoJobType.INDEX) { + if (jobData.type === RepoIndexingJobType.INDEX) { const repo = await this.db.repo.update({ where: { id: jobData.repoId }, data: { @@ -388,7 +388,7 @@ export class RepoIndexManager { logger.info(`Completed index job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); } - else if (jobData.type === RepoJobType.CLEANUP) { + else if (jobData.type === RepoIndexingJobType.CLEANUP) { const repo = await this.db.repo.delete({ where: { id: jobData.repoId }, }); @@ -405,10 +405,10 @@ export class RepoIndexManager { const wasLastAttempt = attempt >= job.opts.attempts; if (wasLastAttempt) { - const { repo } = await this.db.repoJob.update({ + const { repo } = await this.db.repoIndexingJob.update({ where: { id: job.data.jobId }, data: { - status: RepoJobStatus.FAILED, + status: RepoIndexingJobStatus.FAILED, completedAt: new Date(), errorMessage: job.failedReason, }, @@ -428,10 +428,10 @@ export class RepoIndexManager { private onJobStalled = async (jobId: string) => groupmqLifecycleExceptionWrapper('onJobStalled', logger, async () => { const logger = createJobLogger(jobId); - const { repo } = await this.db.repoJob.update({ + const { repo } = await this.db.repoIndexingJob.update({ where: { id: jobId }, data: { - status: RepoJobStatus.FAILED, + status: RepoIndexingJobStatus.FAILED, completedAt: new Date(), errorMessage: 'Job stalled', }, diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index 575e0cc9..aaaad4ea 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -246,7 +246,7 @@ const createGitCloneUrlWithToken = (cloneUrl: string, credentials: { username?: /** * Wraps groupmq worker lifecycle callbacks with exception handling. This prevents - * uncaught exceptions (e.g., like a RepoJob not existing in the DB) from crashing + * uncaught exceptions (e.g., like a RepoIndexingJob not existing in the DB) from crashing * the app. * @see: https://openpanel-dev.github.io/groupmq/api-worker/#events */ diff --git a/packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql b/packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql new file mode 100644 index 00000000..23fc47c3 --- /dev/null +++ b/packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql @@ -0,0 +1,34 @@ +/* + Warnings: + + - You are about to drop the column `repoIndexingStatus` on the `Repo` table. All the data in the column will be lost. + +*/ +-- CreateEnum +CREATE TYPE "RepoIndexingJobStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'); + +-- CreateEnum +CREATE TYPE "RepoIndexingJobType" AS ENUM ('INDEX', 'CLEANUP'); + +-- AlterTable +ALTER TABLE "Repo" DROP COLUMN "repoIndexingStatus"; + +-- DropEnum +DROP TYPE "RepoIndexingStatus"; + +-- CreateTable +CREATE TABLE "RepoIndexingJob" ( + "id" TEXT NOT NULL, + "type" "RepoIndexingJobType" NOT NULL, + "status" "RepoIndexingJobStatus" NOT NULL DEFAULT 'PENDING', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "completedAt" TIMESTAMP(3), + "errorMessage" TEXT, + "repoId" INTEGER NOT NULL, + + CONSTRAINT "RepoIndexingJob_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "RepoIndexingJob" ADD CONSTRAINT "RepoIndexingJob_repoId_fkey" FOREIGN KEY ("repoId") REFERENCES "Repo"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 16ed94f0..8952d0fc 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -10,17 +10,6 @@ datasource db { url = env("DATABASE_URL") } -enum RepoIndexingStatus { - NEW - IN_INDEX_QUEUE - INDEXING - INDEXED - FAILED - IN_GC_QUEUE - GARBAGE_COLLECTING - GARBAGE_COLLECTION_FAILED -} - enum ConnectionSyncStatus { SYNC_NEEDED IN_SYNC_QUEUE @@ -55,14 +44,11 @@ model Repo { connections RepoToConnection[] imageUrl String? - /// @deprecated status tracking is now done via the `jobs` table. - repoIndexingStatus RepoIndexingStatus @default(NEW) - permittedUsers UserToRepoPermission[] permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. - jobs RepoJob[] + jobs RepoIndexingJob[] indexedAt DateTime? /// When the repo was last indexed successfully. external_id String /// The id of the repo in the external service @@ -78,22 +64,22 @@ model Repo { @@index([orgId]) } -enum RepoJobStatus { +enum RepoIndexingJobStatus { PENDING IN_PROGRESS COMPLETED FAILED } -enum RepoJobType { +enum RepoIndexingJobType { INDEX CLEANUP } -model RepoJob { +model RepoIndexingJob { id String @id @default(cuid()) - type RepoJobType - status RepoJobStatus @default(PENDING) + type RepoIndexingJobType + status RepoIndexingJobStatus @default(PENDING) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt completedAt DateTime? diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 4f605d8e..f7ba284a 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -10,7 +10,7 @@ import { prisma } from "@/prisma"; import { render } from "@react-email/components"; import * as Sentry from '@sentry/nextjs'; import { encrypt, generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto"; -import { ApiKey, Org, OrgRole, Prisma, RepoJobStatus, RepoJobType, StripeSubscriptionStatus } from "@sourcebot/db"; +import { ApiKey, Org, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType, StripeSubscriptionStatus } from "@sourcebot/db"; import { createLogger } from "@sourcebot/logger"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; @@ -586,11 +586,11 @@ export const getReposStats = async () => sew(() => orgId: org.id, jobs: { some: { - type: RepoJobType.INDEX, + type: RepoIndexingJobType.INDEX, status: { in: [ - RepoJobStatus.PENDING, - RepoJobStatus.IN_PROGRESS, + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, ] } }, diff --git a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx index 400481c2..54842731 100644 --- a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx @@ -10,7 +10,7 @@ import { env } from "@/env.mjs"; import { ServiceErrorException } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; import { DiscordLogoIcon, GitHubLogoIcon } from "@radix-ui/react-icons"; -import { RepoJobStatus, RepoJobType } from "@sourcebot/db"; +import { RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; import Link from "next/link"; import { redirect } from "next/navigation"; import { OrgSelector } from "../orgSelector"; @@ -43,11 +43,11 @@ export const NavigationMenu = async ({ where: { jobs: { some: { - type: RepoJobType.INDEX, + type: RepoIndexingJobType.INDEX, status: { in: [ - RepoJobStatus.PENDING, - RepoJobStatus.IN_PROGRESS, + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, ] } }, diff --git a/packages/web/src/app/[domain]/repos/page.tsx b/packages/web/src/app/[domain]/repos/page.tsx index c5ed9689..e9dd4e43 100644 --- a/packages/web/src/app/[domain]/repos/page.tsx +++ b/packages/web/src/app/[domain]/repos/page.tsx @@ -1,5 +1,5 @@ import { env } from "@/env.mjs"; -import { RepoJob } from "@sourcebot/db"; +import { RepoIndexingJob } from "@sourcebot/db"; import { Header } from "../components/header"; import { RepoStatus } from "./columns"; import { RepositoryTable } from "./repositoryTable"; @@ -8,7 +8,7 @@ import { withOptionalAuthV2 } from "@/withAuthV2"; import { isServiceError } from "@/lib/utils"; import { ServiceErrorException } from "@/lib/serviceError"; -function getRepoStatus(repo: { indexedAt: Date | null, jobs: RepoJob[] }): RepoStatus { +function getRepoStatus(repo: { indexedAt: Date | null, jobs: RepoIndexingJob[] }): RepoStatus { const latestJob = repo.jobs[0]; if (latestJob?.status === 'PENDING' || latestJob?.status === 'IN_PROGRESS') { From 9faaf8d1ce5db275234629e1f398afa1a6fb4525 Mon Sep 17 00:00:00 2001 From: bkellam Date: Sat, 18 Oct 2025 14:40:29 -0700 Subject: [PATCH 20/24] changelog --- CHANGELOG.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db02263f..4267e59b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed "dubious ownership" errors when cloning / fetching repos. [#553](https://github.com/sourcebot-dev/sourcebot/pull/553) +- Fixed issue with Ask Sourcebot tutorial re-appearing after restarting the browser. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) ### Changed -- Remove spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552) - Improved search performance for unbounded search queries. [#555](https://github.com/sourcebot-dev/sourcebot/pull/555) +- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) +- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) +- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) + +### Removed +- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552) +- Removed connections management page. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) ### Added - Added support for passing db connection url as seperate `DATABASE_HOST`, `DATABASE_USERNAME`, `DATABASE_PASSWORD`, `DATABASE_NAME`, and `DATABASE_ARGS` env vars. [#545](https://github.com/sourcebot-dev/sourcebot/pull/545) From 3b9a504890e5616ab9d4353fe7f6193b0985caa1 Mon Sep 17 00:00:00 2001 From: bkellam Date: Sat, 18 Oct 2025 14:52:21 -0700 Subject: [PATCH 21/24] make repository column width fixed --- packages/web/src/app/[domain]/repos/columns.tsx | 3 +++ packages/web/src/components/ui/data-table.tsx | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/web/src/app/[domain]/repos/columns.tsx b/packages/web/src/app/[domain]/repos/columns.tsx index a1cd1bdc..7d617cda 100644 --- a/packages/web/src/app/[domain]/repos/columns.tsx +++ b/packages/web/src/app/[domain]/repos/columns.tsx @@ -73,6 +73,7 @@ export const columns = (domain: string): ColumnDef[] => [ { accessorKey: "repoDisplayName", header: 'Repository', + size: 500, cell: ({ row: { original: { repoId, repoName, repoDisplayName, imageUrl } } }) => { return (
@@ -110,6 +111,7 @@ export const columns = (domain: string): ColumnDef[] => [ }, { accessorKey: "status", + size: 150, header: ({ column }) => { const uniqueLabels = Object.values(statusLabels); const currentFilter = column.getFilterValue() as string | undefined; @@ -163,6 +165,7 @@ export const columns = (domain: string): ColumnDef[] => [ }, { accessorKey: "lastIndexed", + size: 150, header: ({ column }) => (