diff --git a/.env.development b/.env.development index 1740c2df..9b5b9682 100644 --- a/.env.development +++ b/.env.development @@ -80,9 +80,6 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection # Controls the number of concurrent indexing jobs that can run at once # INDEX_CONCURRENCY_MULTIPLE= -# Controls the polling interval for the web app -# NEXT_PUBLIC_POLLING_INTERVAL_MS= - # Controls the version of the web app # NEXT_PUBLIC_SOURCEBOT_VERSION= diff --git a/CHANGELOG.md b/CHANGELOG.md index db02263f..4267e59b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed "dubious ownership" errors when cloning / fetching repos. [#553](https://github.com/sourcebot-dev/sourcebot/pull/553) +- Fixed issue with Ask Sourcebot tutorial re-appearing after restarting the browser. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) ### Changed -- Remove spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552) - Improved search performance for unbounded search queries. [#555](https://github.com/sourcebot-dev/sourcebot/pull/555) +- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) +- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) +- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) + +### Removed +- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552) +- Removed connections management page. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563) ### Added - Added support for passing db connection url as seperate `DATABASE_HOST`, `DATABASE_USERNAME`, `DATABASE_PASSWORD`, `DATABASE_NAME`, and `DATABASE_ARGS` env vars. [#545](https://github.com/sourcebot-dev/sourcebot/pull/545) diff --git a/package.json b/package.json index c5909e76..7c726c7e 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "cross-env SKIP_ENV_VALIDATION=1 yarn workspaces foreach -A run build", "test": "yarn workspaces foreach -A run test", - "dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web watch:mcp watch:schemas", + "dev": "concurrently --kill-others --names \"zoekt,worker,web,mcp,schemas\" 'yarn dev:zoekt' 'yarn dev:backend' 'yarn dev:web' 'yarn watch:mcp' 'yarn watch:schemas'", "with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --", "dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc", "dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch", @@ -21,9 +21,9 @@ "build:deps": "yarn workspaces foreach -R --from '{@sourcebot/schemas,@sourcebot/error,@sourcebot/crypto,@sourcebot/db,@sourcebot/shared}' run build" }, "devDependencies": { + "concurrently": "^9.2.1", "cross-env": "^7.0.3", - "dotenv-cli": "^8.0.0", - "npm-run-all": "^4.1.5" + "dotenv-cli": "^8.0.0" }, "packageManager": "yarn@4.7.0", "resolutions": { diff --git a/packages/backend/package.json b/packages/backend/package.json index dade7893..800d2504 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -45,6 +45,7 @@ "git-url-parse": "^16.1.0", "gitea-js": "^1.22.0", "glob": "^11.0.0", + "groupmq": "^1.0.0", "ioredis": "^5.4.2", "lowdb": "^7.0.1", "micromatch": "^4.0.8", diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts index e17fce06..ce023fc5 100644 --- a/packages/backend/src/connectionManager.ts +++ b/packages/backend/src/connectionManager.ts @@ -364,12 +364,12 @@ export class ConnectionManager { } } - public dispose() { + public async dispose() { if (this.interval) { clearInterval(this.interval); } - this.worker.close(); - this.queue.close(); + await this.worker.close(); + await this.queue.close(); } } diff --git a/packages/backend/src/constants.ts b/packages/backend/src/constants.ts index 89778fb2..3ede6d4e 100644 --- a/packages/backend/src/constants.ts +++ b/packages/backend/src/constants.ts @@ -1,4 +1,6 @@ +import { env } from "./env.js"; import { Settings } from "./types.js"; +import path from "path"; /** * Default settings. @@ -22,4 +24,7 @@ export const DEFAULT_SETTINGS: Settings = { export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [ 'github', -]; \ No newline at end of file +]; + +export const REPOS_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'repos'); +export const INDEX_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'index'); \ No newline at end of file diff --git a/packages/backend/src/ee/repoPermissionSyncer.ts b/packages/backend/src/ee/repoPermissionSyncer.ts index f411c3e3..453b94f6 100644 --- a/packages/backend/src/ee/repoPermissionSyncer.ts +++ b/packages/backend/src/ee/repoPermissionSyncer.ts @@ -101,12 +101,12 @@ export class RepoPermissionSyncer { }, 1000 * 5); } - public dispose() { + public async dispose() { if (this.interval) { clearInterval(this.interval); } - this.worker.close(); - this.queue.close(); + await this.worker.close(); + await this.queue.close(); } private async schedulePermissionSync(repos: Repo[]) { diff --git a/packages/backend/src/ee/userPermissionSyncer.ts b/packages/backend/src/ee/userPermissionSyncer.ts index 90ae8629..6ef77bcf 100644 --- a/packages/backend/src/ee/userPermissionSyncer.ts +++ b/packages/backend/src/ee/userPermissionSyncer.ts @@ -101,12 +101,12 @@ export class UserPermissionSyncer { }, 1000 * 5); } - public dispose() { + public async dispose() { if (this.interval) { clearInterval(this.interval); } - this.worker.close(); - this.queue.close(); + await this.worker.close(); + await this.queue.close(); } private async schedulePermissionSync(users: User[]) { diff --git a/packages/backend/src/env.ts b/packages/backend/src/env.ts index 80bbba5e..5e6e1e1d 100644 --- a/packages/backend/src/env.ts +++ b/packages/backend/src/env.ts @@ -44,6 +44,7 @@ export const env = createEnv({ LOGTAIL_TOKEN: z.string().optional(), LOGTAIL_HOST: z.string().url().optional(), SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"), + DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'), DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"), CONFIG_PATH: z.string().optional(), diff --git a/packages/backend/src/git.ts b/packages/backend/src/git.ts index c1110625..308f3f4a 100644 --- a/packages/backend/src/git.ts +++ b/packages/backend/src/git.ts @@ -1,30 +1,67 @@ import { CheckRepoActions, GitConfigScope, simpleGit, SimpleGitProgressEvent } from 'simple-git'; import { mkdir } from 'node:fs/promises'; import { env } from './env.js'; +import { dirname, resolve } from 'node:path'; +import { existsSync } from 'node:fs'; type onProgressFn = (event: SimpleGitProgressEvent) => void; +/** + * Creates a simple-git client that has it's working directory + * set to the given path. + */ +const createGitClientForPath = (path: string, onProgress?: onProgressFn, signal?: AbortSignal) => { + if (!existsSync(path)) { + throw new Error(`Path ${path} does not exist`); + } + + const parentPath = resolve(dirname(path)); + + const git = simpleGit({ + progress: onProgress, + abort: signal, + }) + .env({ + ...process.env, + /** + * @note on some inside-baseball on why this is necessary: The specific + * issue we saw was that a `git clone` would fail without throwing, and + * then a subsequent `git config` command would run, but since the clone + * failed, it wouldn't be running in a git directory. Git would then walk + * up the directory tree until it either found a git directory (in the case + * of the development env) or it would hit a GIT_DISCOVERY_ACROSS_FILESYSTEM + * error when trying to cross a filesystem boundary (in the prod case). + * GIT_CEILING_DIRECTORIES ensures that this walk will be limited to the + * parent directory. + */ + GIT_CEILING_DIRECTORIES: parentPath, + }) + .cwd({ + path, + }); + + return git; +} + export const cloneRepository = async ( { cloneUrl, authHeader, path, onProgress, + signal, }: { cloneUrl: string, authHeader?: string, path: string, onProgress?: onProgressFn + signal?: AbortSignal } ) => { try { await mkdir(path, { recursive: true }); - const git = simpleGit({ - progress: onProgress, - }).cwd({ - path, - }) + const git = createGitClientForPath(path, onProgress, signal); const cloneArgs = [ "--bare", @@ -33,7 +70,11 @@ export const cloneRepository = async ( await git.clone(cloneUrl, path, cloneArgs); - await unsetGitConfig(path, ["remote.origin.url"]); + await unsetGitConfig({ + path, + keys: ["remote.origin.url"], + signal, + }); } catch (error: unknown) { const baseLog = `Failed to clone repository: ${path}`; @@ -54,20 +95,17 @@ export const fetchRepository = async ( authHeader, path, onProgress, + signal, }: { cloneUrl: string, authHeader?: string, path: string, - onProgress?: onProgressFn + onProgress?: onProgressFn, + signal?: AbortSignal } ) => { + const git = createGitClientForPath(path, onProgress, signal); try { - const git = simpleGit({ - progress: onProgress, - }).cwd({ - path: path, - }) - if (authHeader) { await git.addConfig("http.extraHeader", authHeader); } @@ -90,12 +128,6 @@ export const fetchRepository = async ( } } finally { if (authHeader) { - const git = simpleGit({ - progress: onProgress, - }).cwd({ - path: path, - }) - await git.raw(["config", "--unset", "http.extraHeader", authHeader]); } } @@ -107,10 +139,19 @@ export const fetchRepository = async ( * that do not exist yet. It will _not_ remove any existing keys that are not * present in gitConfig. */ -export const upsertGitConfig = async (path: string, gitConfig: Record, onProgress?: onProgressFn) => { - const git = simpleGit({ - progress: onProgress, - }).cwd(path); +export const upsertGitConfig = async ( + { + path, + gitConfig, + onProgress, + signal, + }: { + path: string, + gitConfig: Record, + onProgress?: onProgressFn, + signal?: AbortSignal + }) => { + const git = createGitClientForPath(path, onProgress, signal); try { for (const [key, value] of Object.entries(gitConfig)) { @@ -129,10 +170,19 @@ export const upsertGitConfig = async (path: string, gitConfig: Record { - const git = simpleGit({ - progress: onProgress, - }).cwd(path); +export const unsetGitConfig = async ( + { + path, + keys, + onProgress, + signal, + }: { + path: string, + keys: string[], + onProgress?: onProgressFn, + signal?: AbortSignal + }) => { + const git = createGitClientForPath(path, onProgress, signal); try { const configList = await git.listConfig(); @@ -155,10 +205,20 @@ export const unsetGitConfig = async (path: string, keys: string[], onProgress?: /** * Returns true if `path` is the _root_ of a git repository. */ -export const isPathAValidGitRepoRoot = async (path: string, onProgress?: onProgressFn) => { - const git = simpleGit({ - progress: onProgress, - }).cwd(path); +export const isPathAValidGitRepoRoot = async ({ + path, + onProgress, + signal, +}: { + path: string, + onProgress?: onProgressFn, + signal?: AbortSignal +}) => { + if (!existsSync(path)) { + return false; + } + + const git = createGitClientForPath(path, onProgress, signal); try { return git.checkIsRepo(CheckRepoActions.IS_REPO_ROOT); @@ -184,7 +244,7 @@ export const isUrlAValidGitRepo = async (url: string) => { } export const getOriginUrl = async (path: string) => { - const git = simpleGit().cwd(path); + const git = createGitClientForPath(path); try { const remotes = await git.getConfig('remote.origin.url', GitConfigScope.local); @@ -199,18 +259,13 @@ export const getOriginUrl = async (path: string) => { } export const getBranches = async (path: string) => { - const git = simpleGit(); - const branches = await git.cwd({ - path, - }).branch(); - + const git = createGitClientForPath(path); + const branches = await git.branch(); return branches.all; } export const getTags = async (path: string) => { - const git = simpleGit(); - const tags = await git.cwd({ - path, - }).tags(); + const git = createGitClientForPath(path); + const tags = await git.tags(); return tags.all; } \ No newline at end of file diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 93f95e0b..78a1ce6c 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -6,15 +6,13 @@ import { hasEntitlement, loadConfig } from '@sourcebot/shared'; import { existsSync } from 'fs'; import { mkdir } from 'fs/promises'; import { Redis } from 'ioredis'; -import path from 'path'; import { ConnectionManager } from './connectionManager.js'; -import { DEFAULT_SETTINGS } from './constants.js'; -import { env } from "./env.js"; +import { DEFAULT_SETTINGS, INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; import { RepoPermissionSyncer } from './ee/repoPermissionSyncer.js'; -import { PromClient } from './promClient.js'; -import { RepoManager } from './repoManager.js'; -import { AppContext } from "./types.js"; import { UserPermissionSyncer } from "./ee/userPermissionSyncer.js"; +import { env } from "./env.js"; +import { RepoIndexManager } from "./repoIndexManager.js"; +import { PromClient } from './promClient.js'; const logger = createLogger('backend-entrypoint'); @@ -33,9 +31,8 @@ const getSettings = async (configPath?: string) => { } -const cacheDir = env.DATA_CACHE_DIR; -const reposPath = path.join(cacheDir, 'repos'); -const indexPath = path.join(cacheDir, 'index'); +const reposPath = REPOS_CACHE_DIR; +const indexPath = INDEX_CACHE_DIR; if (!existsSync(reposPath)) { await mkdir(reposPath, { recursive: true }); @@ -44,12 +41,6 @@ if (!existsSync(indexPath)) { await mkdir(indexPath, { recursive: true }); } -const context: AppContext = { - indexPath, - reposPath, - cachePath: cacheDir, -} - const prisma = new PrismaClient(); const redis = new Redis(env.REDIS_URL, { @@ -68,14 +59,12 @@ const promClient = new PromClient(); const settings = await getSettings(env.CONFIG_PATH); const connectionManager = new ConnectionManager(prisma, settings, redis); -const repoManager = new RepoManager(prisma, settings, redis, promClient, context); const repoPermissionSyncer = new RepoPermissionSyncer(prisma, settings, redis); const userPermissionSyncer = new UserPermissionSyncer(prisma, settings, redis); - -await repoManager.validateIndexedReposHaveShards(); +const repoIndexManager = new RepoIndexManager(prisma, settings, redis); connectionManager.startScheduler(); -repoManager.startScheduler(); +repoIndexManager.startScheduler(); if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && !hasEntitlement('permission-syncing')) { logger.error('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.'); @@ -87,12 +76,27 @@ else if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && hasEntitlement( } const cleanup = async (signal: string) => { - logger.info(`Recieved ${signal}, cleaning up...`); - - connectionManager.dispose(); - repoManager.dispose(); - repoPermissionSyncer.dispose(); - userPermissionSyncer.dispose(); + logger.info(`Received ${signal}, cleaning up...`); + + const shutdownTimeout = 30000; // 30 seconds + + try { + await Promise.race([ + Promise.all([ + repoIndexManager.dispose(), + connectionManager.dispose(), + repoPermissionSyncer.dispose(), + userPermissionSyncer.dispose(), + promClient.dispose(), + ]), + new Promise((_, reject) => + setTimeout(() => reject(new Error('Shutdown timeout')), shutdownTimeout) + ) + ]); + logger.info('All workers shut down gracefully'); + } catch (error) { + logger.warn('Shutdown timeout or error, forcing exit:', error instanceof Error ? error.message : String(error)); + } await prisma.$disconnect(); await redis.quit(); diff --git a/packages/backend/src/promClient.ts b/packages/backend/src/promClient.ts index 058cfe0b..ba2f3085 100644 --- a/packages/backend/src/promClient.ts +++ b/packages/backend/src/promClient.ts @@ -1,4 +1,5 @@ import express, { Request, Response } from 'express'; +import { Server } from 'http'; import client, { Registry, Counter, Gauge } from 'prom-client'; import { createLogger } from "@sourcebot/logger"; @@ -7,6 +8,8 @@ const logger = createLogger('prometheus-client'); export class PromClient { private registry: Registry; private app: express.Application; + private server: Server; + public activeRepoIndexingJobs: Gauge; public pendingRepoIndexingJobs: Gauge; public repoIndexingReattemptsTotal: Counter; @@ -77,7 +80,7 @@ export class PromClient { help: 'The number of repo garbage collection fails', labelNames: ['repo'], }); - this.registry.registerMetric(this.repoGarbageCollectionFailTotal); + this.registry.registerMetric(this.repoGarbageCollectionFailTotal); this.repoGarbageCollectionSuccessTotal = new Counter({ name: 'repo_garbage_collection_successes', @@ -98,12 +101,17 @@ export class PromClient { res.end(metrics); }); - this.app.listen(this.PORT, () => { + this.server = this.app.listen(this.PORT, () => { logger.info(`Prometheus metrics server is running on port ${this.PORT}`); }); } - getRegistry(): Registry { - return this.registry; + async dispose() { + return new Promise((resolve, reject) => { + this.server.close((err) => { + if (err) reject(err); + else resolve(); + }); + }); } } \ No newline at end of file diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index 098f39c9..80d80ebe 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -497,7 +497,9 @@ export const compileGenericGitHostConfig_file = async ( }; await Promise.all(repoPaths.map(async (repoPath) => { - const isGitRepo = await isPathAValidGitRepoRoot(repoPath); + const isGitRepo = await isPathAValidGitRepoRoot({ + path: repoPath, + }); if (!isGitRepo) { logger.warn(`Skipping ${repoPath} - not a git repository.`); notFound.repos.push(repoPath); diff --git a/packages/backend/src/repoIndexManager.ts b/packages/backend/src/repoIndexManager.ts new file mode 100644 index 00000000..29e07a2b --- /dev/null +++ b/packages/backend/src/repoIndexManager.ts @@ -0,0 +1,456 @@ +import * as Sentry from '@sentry/node'; +import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import { createLogger, Logger } from "@sourcebot/logger"; +import { existsSync } from 'fs'; +import { readdir, rm } from 'fs/promises'; +import { Job, Queue, ReservedJob, Worker } from "groupmq"; +import { Redis } from 'ioredis'; +import { INDEX_CACHE_DIR } from './constants.js'; +import { env } from './env.js'; +import { cloneRepository, fetchRepository, isPathAValidGitRepoRoot, unsetGitConfig, upsertGitConfig } from './git.js'; +import { repoMetadataSchema, RepoWithConnections, Settings } from "./types.js"; +import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure } from './utils.js'; +import { indexGitRepository } from './zoekt.js'; + +const LOG_TAG = 'repo-index-manager'; +const logger = createLogger(LOG_TAG); +const createJobLogger = (jobId: string) => createLogger(`${LOG_TAG}:job:${jobId}`); + +type JobPayload = { + type: 'INDEX' | 'CLEANUP'; + jobId: string; + repoId: number; + repoName: string; +}; + +const JOB_TIMEOUT_MS = 1000 * 60 * 60 * 6; // 6 hour indexing timeout + +/** + * Manages the lifecycle of repository data on disk, including git working copies + * and search index shards. Handles both indexing operations (cloning/fetching repos + * and building search indexes) and cleanup operations (removing orphaned repos and + * their associated data). + * + * Uses a job queue system to process indexing and cleanup tasks asynchronously, + * with configurable concurrency limits and retry logic. Automatically schedules + * re-indexing of repos based on configured intervals and manages garbage collection + * of repos that are no longer connected to any source. + */ +export class RepoIndexManager { + private interval?: NodeJS.Timeout; + private queue: Queue; + private worker: Worker; + + constructor( + private db: PrismaClient, + private settings: Settings, + redis: Redis, + ) { + this.queue = new Queue({ + redis, + namespace: 'repo-index-queue', + jobTimeoutMs: JOB_TIMEOUT_MS, + maxAttempts: 3, + logger: env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true', + }); + + this.worker = new Worker({ + queue: this.queue, + maxStalledCount: 1, + handler: this.runJob.bind(this), + concurrency: this.settings.maxRepoIndexingJobConcurrency, + ...(env.DEBUG_ENABLE_GROUPMQ_LOGGING === 'true' ? { + logger: true, + }: {}), + }); + + this.worker.on('completed', this.onJobCompleted.bind(this)); + this.worker.on('failed', this.onJobFailed.bind(this)); + this.worker.on('stalled', this.onJobStalled.bind(this)); + this.worker.on('error', this.onWorkerError.bind(this)); + } + + public async startScheduler() { + logger.debug('Starting scheduler'); + this.interval = setInterval(async () => { + await this.scheduleIndexJobs(); + await this.scheduleCleanupJobs(); + }, 1000 * 5); + + this.worker.run(); + } + + private async scheduleIndexJobs() { + const thresholdDate = new Date(Date.now() - this.settings.reindexIntervalMs); + const reposToIndex = await this.db.repo.findMany({ + where: { + AND: [ + { + OR: [ + { indexedAt: null }, + { indexedAt: { lt: thresholdDate } }, + ] + }, + { + NOT: { + jobs: { + some: { + AND: [ + { + type: RepoIndexingJobType.INDEX, + }, + { + OR: [ + // Don't schedule if there are active jobs that were created within the threshold date. + // This handles the case where a job is stuck in a pending state and will never be scheduled. + { + AND: [ + { + status: { + in: [ + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, + ] + }, + }, + { + createdAt: { + gt: thresholdDate, + } + } + ] + }, + // Don't schedule if there are recent failed jobs (within the threshold date). + { + AND: [ + { status: RepoIndexingJobStatus.FAILED }, + { completedAt: { gt: thresholdDate } }, + ] + } + ] + } + ] + } + } + } + } + ], + } + }); + + if (reposToIndex.length > 0) { + await this.createJobs(reposToIndex, RepoIndexingJobType.INDEX); + } + } + + private async scheduleCleanupJobs() { + const thresholdDate = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs); + + const reposToCleanup = await this.db.repo.findMany({ + where: { + connections: { + none: {} + }, + OR: [ + { indexedAt: null }, + { indexedAt: { lt: thresholdDate } }, + ], + // Don't schedule if there are active jobs that were created within the threshold date. + NOT: { + jobs: { + some: { + AND: [ + { + type: RepoIndexingJobType.CLEANUP, + }, + { + status: { + in: [ + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, + ] + }, + }, + { + createdAt: { + gt: thresholdDate, + } + } + ] + } + } + } + } + }); + + if (reposToCleanup.length > 0) { + await this.createJobs(reposToCleanup, RepoIndexingJobType.CLEANUP); + } + } + + private async createJobs(repos: Repo[], type: RepoIndexingJobType) { + // @note: we don't perform this in a transaction because + // we want to avoid the situation where a job is created and run + // prior to the transaction being committed. + const jobs = await this.db.repoIndexingJob.createManyAndReturn({ + data: repos.map(repo => ({ + type, + repoId: repo.id, + })), + include: { + repo: true, + } + }); + + for (const job of jobs) { + await this.queue.add({ + groupId: `repo:${job.repoId}`, + data: { + jobId: job.id, + type, + repoName: job.repo.name, + repoId: job.repo.id, + }, + jobId: job.id, + }); + } + } + + private async runJob(job: ReservedJob) { + const id = job.data.jobId; + const logger = createJobLogger(id); + logger.info(`Running ${job.data.type} job ${id} for repo ${job.data.repoName} (id: ${job.data.repoId}) (attempt ${job.attempts + 1} / ${job.maxAttempts})`); + + + const { repo, type: jobType } = await this.db.repoIndexingJob.update({ + where: { + id, + }, + data: { + status: RepoIndexingJobStatus.IN_PROGRESS, + }, + select: { + type: true, + repo: { + include: { + connections: { + include: { + connection: true, + } + } + } + } + } + }); + + const abortController = new AbortController(); + const signalHandler = () => { + logger.info(`Received shutdown signal, aborting...`); + abortController.abort(); // This cancels all operations + }; + + process.on('SIGTERM', signalHandler); + process.on('SIGINT', signalHandler); + + try { + if (jobType === RepoIndexingJobType.INDEX) { + await this.indexRepository(repo, logger, abortController.signal); + } else if (jobType === RepoIndexingJobType.CLEANUP) { + await this.cleanupRepository(repo, logger); + } + } finally { + process.off('SIGTERM', signalHandler); + process.off('SIGINT', signalHandler); + } + } + + private async indexRepository(repo: RepoWithConnections, logger: Logger, signal: AbortSignal) { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + + const metadata = repoMetadataSchema.parse(repo.metadata); + + const credentials = await getAuthCredentialsForRepo(repo, this.db); + const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; + const authHeader = credentials?.authHeader ?? undefined; + + // If the repo path exists but it is not a valid git repository root, this indicates + // that the repository is in a bad state. To fix, we remove the directory and perform + // a fresh clone. + if (existsSync(repoPath) && !(await isPathAValidGitRepoRoot( { path: repoPath } ))) { + const isValidGitRepo = await isPathAValidGitRepoRoot({ + path: repoPath, + signal, + }); + + if (!isValidGitRepo && !isReadOnly) { + logger.warn(`${repoPath} is not a valid git repository root. Deleting directory and performing fresh clone.`); + await rm(repoPath, { recursive: true, force: true }); + } + } + + if (existsSync(repoPath) && !isReadOnly) { + // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ + // write the clone URL (which could contain a auth token) to the + // `remote.origin.url` entry. For the upgrade scenario, we want + // to unset this key since it is no longer needed, hence this line. + // This will no-op if the key is already unset. + // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 + await unsetGitConfig({ + path: repoPath, + keys: ["remote.origin.url"], + signal, + }); + + logger.info(`Fetching ${repo.name} (id: ${repo.id})...`); + const { durationMs } = await measure(() => fetchRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) + }, + signal, + })); + const fetchDuration_s = durationMs / 1000; + + process.stdout.write('\n'); + logger.info(`Fetched ${repo.name} (id: ${repo.id}) in ${fetchDuration_s}s`); + + } else if (!isReadOnly) { + logger.info(`Cloning ${repo.name} (id: ${repo.id})...`); + + const { durationMs } = await measure(() => cloneRepository({ + cloneUrl: cloneUrlMaybeWithToken, + authHeader, + path: repoPath, + onProgress: ({ method, stage, progress }) => { + logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.name} (id: ${repo.id})`) + }, + signal + })); + const cloneDuration_s = durationMs / 1000; + + process.stdout.write('\n'); + logger.info(`Cloned ${repo.name} (id: ${repo.id}) in ${cloneDuration_s}s`); + } + + // Regardless of clone or fetch, always upsert the git config for the repo. + // This ensures that the git config is always up to date for whatever we + // have in the DB. + if (metadata.gitConfig && !isReadOnly) { + await upsertGitConfig({ + path: repoPath, + gitConfig: metadata.gitConfig, + signal, + }); + } + + logger.info(`Indexing ${repo.name} (id: ${repo.id})...`); + const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, signal)); + const indexDuration_s = durationMs / 1000; + logger.info(`Indexed ${repo.name} (id: ${repo.id}) in ${indexDuration_s}s`); + } + + private async cleanupRepository(repo: Repo, logger: Logger) { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + if (existsSync(repoPath) && !isReadOnly) { + logger.info(`Deleting repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + const shardPrefix = getShardPrefix(repo.orgId, repo.id); + const files = (await readdir(INDEX_CACHE_DIR)).filter(file => file.startsWith(shardPrefix)); + for (const file of files) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.info(`Deleting shard file ${filePath}`); + await rm(filePath, { force: true }); + } + } + + private onJobCompleted = async (job: Job) => + groupmqLifecycleExceptionWrapper('onJobCompleted', logger, async () => { + const logger = createJobLogger(job.data.jobId); + const jobData = await this.db.repoIndexingJob.update({ + where: { id: job.data.jobId }, + data: { + status: RepoIndexingJobStatus.COMPLETED, + completedAt: new Date(), + } + }); + + if (jobData.type === RepoIndexingJobType.INDEX) { + const repo = await this.db.repo.update({ + where: { id: jobData.repoId }, + data: { + indexedAt: new Date(), + } + }); + + logger.info(`Completed index job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); + } + else if (jobData.type === RepoIndexingJobType.CLEANUP) { + const repo = await this.db.repo.delete({ + where: { id: jobData.repoId }, + }); + + logger.info(`Completed cleanup job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id})`); + } + }); + + private onJobFailed = async (job: Job) => + groupmqLifecycleExceptionWrapper('onJobFailed', logger, async () => { + const logger = createJobLogger(job.data.jobId); + + const attempt = job.attemptsMade + 1; + const wasLastAttempt = attempt >= job.opts.attempts; + + if (wasLastAttempt) { + const { repo } = await this.db.repoIndexingJob.update({ + where: { id: job.data.jobId }, + data: { + status: RepoIndexingJobStatus.FAILED, + completedAt: new Date(), + errorMessage: job.failedReason, + }, + select: { repo: true } + }); + + logger.error(`Failed job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id}). Attempt ${attempt} / ${job.opts.attempts}. Failing job.`); + } else { + const repo = await this.db.repo.findUniqueOrThrow({ + where: { id: job.data.repoId }, + }); + + logger.warn(`Failed job ${job.data.jobId} for repo ${repo.name} (id: ${repo.id}). Attempt ${attempt} / ${job.opts.attempts}. Retrying.`); + } + }); + + private onJobStalled = async (jobId: string) => + groupmqLifecycleExceptionWrapper('onJobStalled', logger, async () => { + const logger = createJobLogger(jobId); + const { repo } = await this.db.repoIndexingJob.update({ + where: { id: jobId }, + data: { + status: RepoIndexingJobStatus.FAILED, + completedAt: new Date(), + errorMessage: 'Job stalled', + }, + select: { repo: true } + }); + + logger.error(`Job ${jobId} stalled for repo ${repo.name} (id: ${repo.id})`); + }); + + private async onWorkerError(error: Error) { + Sentry.captureException(error); + logger.error(`Index syncer worker error.`, error); + } + + public async dispose() { + if (this.interval) { + clearInterval(this.interval); + } + await this.worker.close(); + await this.queue.close(); + } +} \ No newline at end of file diff --git a/packages/backend/src/repoManager.ts b/packages/backend/src/repoManager.ts deleted file mode 100644 index 89e41673..00000000 --- a/packages/backend/src/repoManager.ts +++ /dev/null @@ -1,566 +0,0 @@ -import * as Sentry from "@sentry/node"; -import { PrismaClient, Repo, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db"; -import { createLogger } from "@sourcebot/logger"; -import { Job, Queue, Worker } from 'bullmq'; -import { existsSync, promises, readdirSync } from 'fs'; -import { Redis } from 'ioredis'; -import { env } from './env.js'; -import { cloneRepository, fetchRepository, unsetGitConfig, upsertGitConfig } from "./git.js"; -import { PromClient } from './promClient.js'; -import { AppContext, RepoWithConnections, Settings, repoMetadataSchema } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, measure } from "./utils.js"; -import { indexGitRepository } from "./zoekt.js"; - -const REPO_INDEXING_QUEUE = 'repoIndexingQueue'; -const REPO_GC_QUEUE = 'repoGarbageCollectionQueue'; - -type RepoIndexingPayload = { - repo: RepoWithConnections, -} - -type RepoGarbageCollectionPayload = { - repo: Repo, -} - -const logger = createLogger('repo-manager'); - -export class RepoManager { - private indexWorker: Worker; - private indexQueue: Queue; - private gcWorker: Worker; - private gcQueue: Queue; - private interval?: NodeJS.Timeout; - - constructor( - private db: PrismaClient, - private settings: Settings, - redis: Redis, - private promClient: PromClient, - private ctx: AppContext, - ) { - // Repo indexing - this.indexQueue = new Queue(REPO_INDEXING_QUEUE, { - connection: redis, - }); - this.indexWorker = new Worker(REPO_INDEXING_QUEUE, this.runIndexJob.bind(this), { - connection: redis, - concurrency: this.settings.maxRepoIndexingJobConcurrency, - }); - this.indexWorker.on('completed', this.onIndexJobCompleted.bind(this)); - this.indexWorker.on('failed', this.onIndexJobFailed.bind(this)); - - // Garbage collection - this.gcQueue = new Queue(REPO_GC_QUEUE, { - connection: redis, - }); - this.gcWorker = new Worker(REPO_GC_QUEUE, this.runGarbageCollectionJob.bind(this), { - connection: redis, - concurrency: this.settings.maxRepoGarbageCollectionJobConcurrency, - }); - this.gcWorker.on('completed', this.onGarbageCollectionJobCompleted.bind(this)); - this.gcWorker.on('failed', this.onGarbageCollectionJobFailed.bind(this)); - } - - public startScheduler() { - logger.debug('Starting scheduler'); - this.interval = setInterval(async () => { - await this.fetchAndScheduleRepoIndexing(); - await this.fetchAndScheduleRepoGarbageCollection(); - await this.fetchAndScheduleRepoTimeouts(); - }, this.settings.reindexRepoPollingIntervalMs); - } - - /////////////////////////// - // Repo indexing - /////////////////////////// - - private async scheduleRepoIndexingBulk(repos: RepoWithConnections[]) { - await this.db.$transaction(async (tx) => { - await tx.repo.updateMany({ - where: { id: { in: repos.map(repo => repo.id) } }, - data: { repoIndexingStatus: RepoIndexingStatus.IN_INDEX_QUEUE } - }); - - const reposByOrg = repos.reduce>((acc, repo) => { - if (!acc[repo.orgId]) { - acc[repo.orgId] = []; - } - acc[repo.orgId].push(repo); - return acc; - }, {}); - - for (const orgId in reposByOrg) { - const orgRepos = reposByOrg[orgId]; - // Set priority based on number of repos (more repos = lower priority) - // This helps prevent large orgs from overwhelming the indexQueue - const priority = Math.min(Math.ceil(orgRepos.length / 10), 2097152); - - await this.indexQueue.addBulk(orgRepos.map(repo => ({ - name: 'repoIndexJob', - data: { repo }, - opts: { - priority: priority, - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - }, - }))); - - // Increment pending jobs counter for each repo added - orgRepos.forEach(repo => { - this.promClient.pendingRepoIndexingJobs.inc({ repo: repo.id.toString() }); - }); - - logger.info(`Added ${orgRepos.length} jobs to indexQueue for org ${orgId} with priority ${priority}`); - } - - - }).catch((err: unknown) => { - logger.error(`Failed to add jobs to indexQueue for repos ${repos.map(repo => repo.id).join(', ')}: ${err}`); - }); - } - - - private async fetchAndScheduleRepoIndexing() { - const thresholdDate = new Date(Date.now() - this.settings.reindexIntervalMs); - const repos = await this.db.repo.findMany({ - where: { - OR: [ - // "NEW" is really a misnomer here - it just means that the repo needs to be indexed - // immediately. In most cases, this will be because the repo was just created and - // is indeed "new". However, it could also be that a "retry" was requested on a failed - // index. So, we don't want to block on the indexedAt timestamp here. - { - repoIndexingStatus: RepoIndexingStatus.NEW, - }, - // When the repo has already been indexed, we only want to reindex if the reindexing - // interval has elapsed (or if the date isn't set for some reason). - { - AND: [ - { repoIndexingStatus: RepoIndexingStatus.INDEXED }, - { - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } }, - ] - } - ] - } - ] - }, - include: { - connections: { - include: { - connection: true - } - } - } - }); - - if (repos.length > 0) { - await this.scheduleRepoIndexingBulk(repos); - } - } - - private async syncGitRepository(repo: RepoWithConnections, repoAlreadyInIndexingState: boolean) { - const { path: repoPath, isReadOnly } = getRepoPath(repo, this.ctx); - - const metadata = repoMetadataSchema.parse(repo.metadata); - - // If the repo was already in the indexing state, this job was likely killed and picked up again. As a result, - // to ensure the repo state is valid, we delete the repo if it exists so we get a fresh clone - if (repoAlreadyInIndexingState && existsSync(repoPath) && !isReadOnly) { - logger.info(`Deleting repo directory ${repoPath} during sync because it was already in the indexing state`); - await promises.rm(repoPath, { recursive: true, force: true }); - } - - const credentials = await getAuthCredentialsForRepo(repo, this.db); - const cloneUrlMaybeWithToken = credentials?.cloneUrlWithToken ?? repo.cloneUrl; - const authHeader = credentials?.authHeader ?? undefined; - - if (existsSync(repoPath) && !isReadOnly) { - // @NOTE: in #483, we changed the cloning method s.t., we _no longer_ - // write the clone URL (which could contain a auth token) to the - // `remote.origin.url` entry. For the upgrade scenario, we want - // to unset this key since it is no longer needed, hence this line. - // This will no-op if the key is already unset. - // @see: https://github.com/sourcebot-dev/sourcebot/pull/483 - await unsetGitConfig(repoPath, ["remote.origin.url"]); - - logger.info(`Fetching ${repo.displayName}...`); - const { durationMs } = await measure(() => fetchRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`) - } - })); - const fetchDuration_s = durationMs / 1000; - - process.stdout.write('\n'); - logger.info(`Fetched ${repo.displayName} in ${fetchDuration_s}s`); - - } else if (!isReadOnly) { - logger.info(`Cloning ${repo.displayName}...`); - - const { durationMs } = await measure(() => cloneRepository({ - cloneUrl: cloneUrlMaybeWithToken, - authHeader, - path: repoPath, - onProgress: ({ method, stage, progress }) => { - logger.debug(`git.${method} ${stage} stage ${progress}% complete for ${repo.displayName}`) - } - })); - const cloneDuration_s = durationMs / 1000; - - process.stdout.write('\n'); - logger.info(`Cloned ${repo.displayName} in ${cloneDuration_s}s`); - } - - // Regardless of clone or fetch, always upsert the git config for the repo. - // This ensures that the git config is always up to date for whatever we - // have in the DB. - if (metadata.gitConfig && !isReadOnly) { - await upsertGitConfig(repoPath, metadata.gitConfig); - } - - logger.info(`Indexing ${repo.displayName}...`); - const { durationMs } = await measure(() => indexGitRepository(repo, this.settings, this.ctx)); - const indexDuration_s = durationMs / 1000; - logger.info(`Indexed ${repo.displayName} in ${indexDuration_s}s`); - } - - private async runIndexJob(job: Job) { - logger.info(`Running index job (id: ${job.id}) for repo ${job.data.repo.displayName}`); - const repo = job.data.repo as RepoWithConnections; - - // We have to use the existing repo object to get the repoIndexingStatus because the repo object - // inside the job is unchanged from when it was added to the queue. - const existingRepo = await this.db.repo.findUnique({ - where: { - id: repo.id, - }, - }); - if (!existingRepo) { - logger.error(`Repo ${repo.id} not found`); - const e = new Error(`Repo ${repo.id} not found`); - Sentry.captureException(e); - throw e; - } - const repoAlreadyInIndexingState = existingRepo.repoIndexingStatus === RepoIndexingStatus.INDEXING; - - - await this.db.repo.update({ - where: { - id: repo.id, - }, - data: { - repoIndexingStatus: RepoIndexingStatus.INDEXING, - } - }); - this.promClient.activeRepoIndexingJobs.inc(); - this.promClient.pendingRepoIndexingJobs.dec({ repo: repo.id.toString() }); - - let attempts = 0; - const maxAttempts = 3; - - while (attempts < maxAttempts) { - try { - await this.syncGitRepository(repo, repoAlreadyInIndexingState); - break; - } catch (error) { - Sentry.captureException(error); - - attempts++; - this.promClient.repoIndexingReattemptsTotal.inc(); - if (attempts === maxAttempts) { - logger.error(`Failed to sync repository ${repo.name} (id: ${repo.id}) after ${maxAttempts} attempts. Error: ${error}`); - throw error; - } - - const sleepDuration = (env.REPO_SYNC_RETRY_BASE_SLEEP_SECONDS * 1000) * Math.pow(2, attempts - 1); - logger.error(`Failed to sync repository ${repo.name} (id: ${repo.id}), attempt ${attempts}/${maxAttempts}. Sleeping for ${sleepDuration / 1000}s... Error: ${error}`); - await new Promise(resolve => setTimeout(resolve, sleepDuration)); - } - } - } - - private async onIndexJobCompleted(job: Job) { - logger.info(`Repo index job for repo ${job.data.repo.displayName} (id: ${job.data.repo.id}, jobId: ${job.id}) completed`); - this.promClient.activeRepoIndexingJobs.dec(); - this.promClient.repoIndexingSuccessTotal.inc(); - - await this.db.repo.update({ - where: { - id: job.data.repo.id, - }, - data: { - indexedAt: new Date(), - repoIndexingStatus: RepoIndexingStatus.INDEXED, - } - }); - } - - private async onIndexJobFailed(job: Job | undefined, err: unknown) { - logger.info(`Repo index job for repo ${job?.data.repo.displayName} (id: ${job?.data.repo.id}, jobId: ${job?.id}) failed with error: ${err}`); - Sentry.captureException(err, { - tags: { - repoId: job?.data.repo.id, - jobId: job?.id, - queue: REPO_INDEXING_QUEUE, - } - }); - - if (job) { - this.promClient.activeRepoIndexingJobs.dec(); - this.promClient.repoIndexingFailTotal.inc(); - - await this.db.repo.update({ - where: { - id: job.data.repo.id, - }, - data: { - repoIndexingStatus: RepoIndexingStatus.FAILED, - } - }) - } - } - - /////////////////////////// - // Repo garbage collection - /////////////////////////// - - private async scheduleRepoGarbageCollectionBulk(repos: Repo[]) { - await this.db.$transaction(async (tx) => { - await tx.repo.updateMany({ - where: { id: { in: repos.map(repo => repo.id) } }, - data: { repoIndexingStatus: RepoIndexingStatus.IN_GC_QUEUE } - }); - - await this.gcQueue.addBulk(repos.map(repo => ({ - name: 'repoGarbageCollectionJob', - data: { repo }, - opts: { - removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE, - removeOnFail: env.REDIS_REMOVE_ON_FAIL, - } - }))); - - logger.info(`Added ${repos.length} jobs to gcQueue`); - }); - } - - private async fetchAndScheduleRepoGarbageCollection() { - //////////////////////////////////// - // Get repos with no connections - //////////////////////////////////// - - - const thresholdDate = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs); - const reposWithNoConnections = await this.db.repo.findMany({ - where: { - repoIndexingStatus: { - in: [ - RepoIndexingStatus.INDEXED, // we don't include NEW repos here because they'll be picked up by the index queue (potential race condition) - RepoIndexingStatus.FAILED, - ] - }, - connections: { - none: {} - }, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } } - ] - }, - }); - if (reposWithNoConnections.length > 0) { - logger.info(`Garbage collecting ${reposWithNoConnections.length} repos with no connections: ${reposWithNoConnections.map(repo => repo.id).join(', ')}`); - } - - //////////////////////////////////// - // Get inactive org repos - //////////////////////////////////// - const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); - const inactiveOrgRepos = await this.db.repo.findMany({ - where: { - org: { - stripeSubscriptionStatus: StripeSubscriptionStatus.INACTIVE, - stripeLastUpdatedAt: { - lt: sevenDaysAgo - } - }, - OR: [ - { indexedAt: null }, - { indexedAt: { lt: thresholdDate } } - ] - } - }); - - if (inactiveOrgRepos.length > 0) { - logger.info(`Garbage collecting ${inactiveOrgRepos.length} inactive org repos: ${inactiveOrgRepos.map(repo => repo.id).join(', ')}`); - } - - const reposToDelete = [...reposWithNoConnections, ...inactiveOrgRepos]; - if (reposToDelete.length > 0) { - await this.scheduleRepoGarbageCollectionBulk(reposToDelete); - } - } - - private async runGarbageCollectionJob(job: Job) { - logger.info(`Running garbage collection job (id: ${job.id}) for repo ${job.data.repo.displayName} (id: ${job.data.repo.id})`); - this.promClient.activeRepoGarbageCollectionJobs.inc(); - - const repo = job.data.repo as Repo; - await this.db.repo.update({ - where: { - id: repo.id - }, - data: { - repoIndexingStatus: RepoIndexingStatus.GARBAGE_COLLECTING - } - }); - - // delete cloned repo - const { path: repoPath, isReadOnly } = getRepoPath(repo, this.ctx); - if (existsSync(repoPath) && !isReadOnly) { - logger.info(`Deleting repo directory ${repoPath}`); - await promises.rm(repoPath, { recursive: true, force: true }); - } - - // delete shards - const shardPrefix = getShardPrefix(repo.orgId, repo.id); - const files = readdirSync(this.ctx.indexPath).filter(file => file.startsWith(shardPrefix)); - for (const file of files) { - const filePath = `${this.ctx.indexPath}/${file}`; - logger.info(`Deleting shard file ${filePath}`); - await promises.rm(filePath, { force: true }); - } - } - - private async onGarbageCollectionJobCompleted(job: Job) { - logger.info(`Garbage collection job ${job.id} completed`); - this.promClient.activeRepoGarbageCollectionJobs.dec(); - this.promClient.repoGarbageCollectionSuccessTotal.inc(); - - await this.db.repo.delete({ - where: { - id: job.data.repo.id - } - }); - } - - private async onGarbageCollectionJobFailed(job: Job | undefined, err: unknown) { - logger.info(`Garbage collection job failed (id: ${job?.id ?? 'unknown'}) with error: ${err}`); - Sentry.captureException(err, { - tags: { - repoId: job?.data.repo.id, - jobId: job?.id, - queue: REPO_GC_QUEUE, - } - }); - - if (job) { - this.promClient.activeRepoGarbageCollectionJobs.dec(); - this.promClient.repoGarbageCollectionFailTotal.inc(); - - await this.db.repo.update({ - where: { - id: job.data.repo.id - }, - data: { - repoIndexingStatus: RepoIndexingStatus.GARBAGE_COLLECTION_FAILED - } - }); - } - } - - /////////////////////////// - // Repo index validation - /////////////////////////// - - public async validateIndexedReposHaveShards() { - logger.info('Validating indexed repos have shards...'); - - const indexedRepos = await this.db.repo.findMany({ - where: { - repoIndexingStatus: RepoIndexingStatus.INDEXED - } - }); - logger.info(`Found ${indexedRepos.length} repos in the DB marked as INDEXED`); - - if (indexedRepos.length === 0) { - return; - } - - const files = readdirSync(this.ctx.indexPath); - const reposToReindex: number[] = []; - for (const repo of indexedRepos) { - const shardPrefix = getShardPrefix(repo.orgId, repo.id); - - // TODO: this doesn't take into account if a repo has multiple shards and only some of them are missing. To support that, this logic - // would need to know how many total shards are expected for this repo - let hasShards = false; - try { - hasShards = files.some(file => file.startsWith(shardPrefix)); - } catch (error) { - logger.error(`Failed to read index directory ${this.ctx.indexPath}: ${error}`); - continue; - } - - if (!hasShards) { - logger.info(`Repo ${repo.displayName} (id: ${repo.id}) is marked as INDEXED but has no shards on disk. Marking for reindexing.`); - reposToReindex.push(repo.id); - } - } - - if (reposToReindex.length > 0) { - await this.db.repo.updateMany({ - where: { - id: { in: reposToReindex } - }, - data: { - repoIndexingStatus: RepoIndexingStatus.NEW - } - }); - logger.info(`Marked ${reposToReindex.length} repos for reindexing due to missing shards`); - } - - logger.info('Done validating indexed repos have shards'); - } - - private async fetchAndScheduleRepoTimeouts() { - const repos = await this.db.repo.findMany({ - where: { - repoIndexingStatus: RepoIndexingStatus.INDEXING, - updatedAt: { - lt: new Date(Date.now() - this.settings.repoIndexTimeoutMs) - } - } - }); - - if (repos.length > 0) { - logger.info(`Scheduling ${repos.length} repo timeouts`); - await this.scheduleRepoTimeoutsBulk(repos); - } - } - - private async scheduleRepoTimeoutsBulk(repos: Repo[]) { - await this.db.$transaction(async (tx) => { - await tx.repo.updateMany({ - where: { id: { in: repos.map(repo => repo.id) } }, - data: { repoIndexingStatus: RepoIndexingStatus.FAILED } - }); - }); - } - - public async dispose() { - if (this.interval) { - clearInterval(this.interval); - } - this.indexWorker.close(); - this.indexQueue.close(); - this.gcQueue.close(); - this.gcWorker.close(); - } -} \ No newline at end of file diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 2ea42d04..8e27867d 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -2,20 +2,6 @@ import { Connection, Repo, RepoToConnection } from "@sourcebot/db"; import { Settings as SettingsSchema } from "@sourcebot/schemas/v3/index.type"; import { z } from "zod"; -export type AppContext = { - /** - * Path to the repos cache directory. - */ - reposPath: string; - - /** - * Path to the index cache directory; - */ - indexPath: string; - - cachePath: string; -} - export type Settings = Required; // Structure of the `metadata` field in the `Repo` table. diff --git a/packages/backend/src/utils.ts b/packages/backend/src/utils.ts index e6ac5f93..aaaad4ea 100644 --- a/packages/backend/src/utils.ts +++ b/packages/backend/src/utils.ts @@ -1,11 +1,12 @@ import { Logger } from "winston"; -import { AppContext, RepoAuthCredentials, RepoWithConnections } from "./types.js"; +import { RepoAuthCredentials, RepoWithConnections } from "./types.js"; import path from 'path'; import { PrismaClient, Repo } from "@sourcebot/db"; import { getTokenFromConfig as getTokenFromConfigBase } from "@sourcebot/crypto"; import { BackendException, BackendError } from "@sourcebot/error"; import * as Sentry from "@sentry/node"; import { GithubConnectionConfig, GitlabConnectionConfig, GiteaConnectionConfig, BitbucketConnectionConfig, AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/connection.type'; +import { REPOS_CACHE_DIR } from "./constants.js"; export const measure = async (cb: () => Promise) => { const start = Date.now(); @@ -69,7 +70,7 @@ export const arraysEqualShallow = (a?: readonly T[], b?: readonly T[]) => { // @note: this function is duplicated in `packages/web/src/features/fileTree/actions.ts`. // @todo: we should move this to a shared package. -export const getRepoPath = (repo: Repo, ctx: AppContext): { path: string, isReadOnly: boolean } => { +export const getRepoPath = (repo: Repo): { path: string, isReadOnly: boolean } => { // If we are dealing with a local repository, then use that as the path. // Mark as read-only since we aren't guaranteed to have write access to the local filesystem. const cloneUrl = new URL(repo.cloneUrl); @@ -81,7 +82,7 @@ export const getRepoPath = (repo: Repo, ctx: AppContext): { path: string, isRead } return { - path: path.join(ctx.reposPath, repo.id.toString()), + path: path.join(REPOS_CACHE_DIR, repo.id.toString()), isReadOnly: false, } } @@ -241,3 +242,20 @@ const createGitCloneUrlWithToken = (cloneUrl: string, credentials: { username?: } return url.toString(); } + + +/** + * Wraps groupmq worker lifecycle callbacks with exception handling. This prevents + * uncaught exceptions (e.g., like a RepoIndexingJob not existing in the DB) from crashing + * the app. + * @see: https://openpanel-dev.github.io/groupmq/api-worker/#events + */ +export const groupmqLifecycleExceptionWrapper = async (name: string, logger: Logger, fn: () => Promise) => { + try { + await fn(); + } catch (error) { + Sentry.captureException(error); + logger.error(`Exception thrown while executing lifecycle function \`${name}\`.`, error); + } +} + diff --git a/packages/backend/src/zoekt.ts b/packages/backend/src/zoekt.ts index 076820c9..ad75927a 100644 --- a/packages/backend/src/zoekt.ts +++ b/packages/backend/src/zoekt.ts @@ -1,21 +1,21 @@ -import { exec } from "child_process"; -import { AppContext, repoMetadataSchema, Settings } from "./types.js"; import { Repo } from "@sourcebot/db"; -import { getRepoPath } from "./utils.js"; -import { getShardPrefix } from "./utils.js"; -import { getBranches, getTags } from "./git.js"; -import micromatch from "micromatch"; import { createLogger } from "@sourcebot/logger"; +import { exec } from "child_process"; +import micromatch from "micromatch"; +import { INDEX_CACHE_DIR } from "./constants.js"; +import { getBranches, getTags } from "./git.js"; import { captureEvent } from "./posthog.js"; +import { repoMetadataSchema, Settings } from "./types.js"; +import { getRepoPath, getShardPrefix } from "./utils.js"; const logger = createLogger('zoekt'); -export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: AppContext) => { +export const indexGitRepository = async (repo: Repo, settings: Settings, signal?: AbortSignal) => { let revisions = [ 'HEAD' ]; - const { path: repoPath } = getRepoPath(repo, ctx); + const { path: repoPath } = getRepoPath(repo); const shardPrefix = getShardPrefix(repo.orgId, repo.id); const metadata = repoMetadataSchema.parse(repo.metadata); @@ -60,7 +60,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: Ap const command = [ 'zoekt-git-index', '-allow_missing_branches', - `-index ${ctx.indexPath}`, + `-index ${INDEX_CACHE_DIR}`, `-max_trigram_count ${settings.maxTrigramCount}`, `-file_limit ${settings.maxFileSize}`, `-branches "${revisions.join(',')}"`, @@ -71,7 +71,7 @@ export const indexGitRepository = async (repo: Repo, settings: Settings, ctx: Ap ].join(' '); return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => { - exec(command, (error, stdout, stderr) => { + exec(command, { signal }, (error, stdout, stderr) => { if (error) { reject(error); return; diff --git a/packages/backend/vitest.config.ts b/packages/backend/vitest.config.ts index 7c052526..5b2ab0d5 100644 --- a/packages/backend/vitest.config.ts +++ b/packages/backend/vitest.config.ts @@ -4,5 +4,8 @@ export default defineConfig({ test: { environment: 'node', watch: false, + env: { + DATA_CACHE_DIR: 'test-data' + } } }); \ No newline at end of file diff --git a/packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql b/packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql new file mode 100644 index 00000000..23fc47c3 --- /dev/null +++ b/packages/db/prisma/migrations/20251018212113_add_repo_indexing_job_table/migration.sql @@ -0,0 +1,34 @@ +/* + Warnings: + + - You are about to drop the column `repoIndexingStatus` on the `Repo` table. All the data in the column will be lost. + +*/ +-- CreateEnum +CREATE TYPE "RepoIndexingJobStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'); + +-- CreateEnum +CREATE TYPE "RepoIndexingJobType" AS ENUM ('INDEX', 'CLEANUP'); + +-- AlterTable +ALTER TABLE "Repo" DROP COLUMN "repoIndexingStatus"; + +-- DropEnum +DROP TYPE "RepoIndexingStatus"; + +-- CreateTable +CREATE TABLE "RepoIndexingJob" ( + "id" TEXT NOT NULL, + "type" "RepoIndexingJobType" NOT NULL, + "status" "RepoIndexingJobStatus" NOT NULL DEFAULT 'PENDING', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + "completedAt" TIMESTAMP(3), + "errorMessage" TEXT, + "repoId" INTEGER NOT NULL, + + CONSTRAINT "RepoIndexingJob_pkey" PRIMARY KEY ("id") +); + +-- AddForeignKey +ALTER TABLE "RepoIndexingJob" ADD CONSTRAINT "RepoIndexingJob_repoId_fkey" FOREIGN KEY ("repoId") REFERENCES "Repo"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index bdebbc69..8952d0fc 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -10,17 +10,6 @@ datasource db { url = env("DATABASE_URL") } -enum RepoIndexingStatus { - NEW - IN_INDEX_QUEUE - INDEXING - INDEXED - FAILED - IN_GC_QUEUE - GARBAGE_COLLECTING - GARBAGE_COLLECTION_FAILED -} - enum ConnectionSyncStatus { SYNC_NEEDED IN_SYNC_QUEUE @@ -46,7 +35,6 @@ model Repo { displayName String? /// Display name of the repo for UI (ex. sourcebot-dev/sourcebot) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - indexedAt DateTime? /// When the repo was last indexed successfully. isFork Boolean isArchived Boolean isPublic Boolean @default(false) @@ -55,12 +43,14 @@ model Repo { webUrl String? connections RepoToConnection[] imageUrl String? - repoIndexingStatus RepoIndexingStatus @default(NEW) permittedUsers UserToRepoPermission[] permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. + jobs RepoIndexingJob[] + indexedAt DateTime? /// When the repo was last indexed successfully. + external_id String /// The id of the repo in the external service external_codeHostType String /// The type of the external service (e.g., github, gitlab, etc.) external_codeHostUrl String /// The base url of the external service (e.g., https://github.com) @@ -74,6 +64,32 @@ model Repo { @@index([orgId]) } +enum RepoIndexingJobStatus { + PENDING + IN_PROGRESS + COMPLETED + FAILED +} + +enum RepoIndexingJobType { + INDEX + CLEANUP +} + +model RepoIndexingJob { + id String @id @default(cuid()) + type RepoIndexingJobType + status RepoIndexingJobStatus @default(PENDING) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + completedAt DateTime? + + errorMessage String? + + repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) + repoId Int +} + enum RepoPermissionSyncJobStatus { PENDING IN_PROGRESS diff --git a/packages/logger/src/index.ts b/packages/logger/src/index.ts index d3998d2c..635c8b3c 100644 --- a/packages/logger/src/index.ts +++ b/packages/logger/src/index.ts @@ -1,4 +1,4 @@ -import winston, { format } from 'winston'; +import winston, { format, Logger } from 'winston'; import { Logtail } from '@logtail/node'; import { LogtailTransport } from '@logtail/winston'; import { MESSAGE } from 'triple-beam'; @@ -48,7 +48,7 @@ const createLogger = (label: string) => { format: combine( errors({ stack: true }), timestamp(), - labelFn({ label: label }) + labelFn({ label: label }), ), transports: [ new winston.transports.Console({ @@ -84,4 +84,8 @@ const createLogger = (label: string) => { export { createLogger -}; \ No newline at end of file +}; + +export type { + Logger, +} \ No newline at end of file diff --git a/packages/mcp/src/schemas.ts b/packages/mcp/src/schemas.ts index 0bb8ff9a..b477e8f1 100644 --- a/packages/mcp/src/schemas.ts +++ b/packages/mcp/src/schemas.ts @@ -143,17 +143,6 @@ export const searchResponseSchema = z.object({ isSearchExhaustive: z.boolean(), }); -enum RepoIndexingStatus { - NEW = 'NEW', - IN_INDEX_QUEUE = 'IN_INDEX_QUEUE', - INDEXING = 'INDEXING', - INDEXED = 'INDEXED', - FAILED = 'FAILED', - IN_GC_QUEUE = 'IN_GC_QUEUE', - GARBAGE_COLLECTING = 'GARBAGE_COLLECTING', - GARBAGE_COLLECTION_FAILED = 'GARBAGE_COLLECTION_FAILED' -} - export const repositoryQuerySchema = z.object({ codeHostType: z.string(), repoId: z.number(), @@ -163,7 +152,6 @@ export const repositoryQuerySchema = z.object({ webUrl: z.string().optional(), imageUrl: z.string().optional(), indexedAt: z.coerce.date().optional(), - repoIndexingStatus: z.nativeEnum(RepoIndexingStatus), }); export const listRepositoriesResponseSchema = repositoryQuerySchema.array(); diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 5b73922c..f7ba284a 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -5,46 +5,32 @@ import { env } from "@/env.mjs"; import { addUserToOrganization, orgHasAvailability } from "@/lib/authUtils"; import { ErrorCode } from "@/lib/errorCodes"; import { notAuthenticated, notFound, orgNotFound, secretAlreadyExists, ServiceError, ServiceErrorException, unexpectedError } from "@/lib/serviceError"; -import { CodeHostType, getOrgMetadata, isHttpError, isServiceError } from "@/lib/utils"; +import { getOrgMetadata, isHttpError, isServiceError } from "@/lib/utils"; import { prisma } from "@/prisma"; import { render } from "@react-email/components"; import * as Sentry from '@sentry/nextjs'; -import { decrypt, encrypt, generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto"; -import { ApiKey, ConnectionSyncStatus, Org, OrgRole, Prisma, RepoIndexingStatus, StripeSubscriptionStatus } from "@sourcebot/db"; +import { encrypt, generateApiKey, getTokenFromConfig, hashSecret } from "@sourcebot/crypto"; +import { ApiKey, Org, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType, StripeSubscriptionStatus } from "@sourcebot/db"; import { createLogger } from "@sourcebot/logger"; -import { azuredevopsSchema } from "@sourcebot/schemas/v3/azuredevops.schema"; -import { bitbucketSchema } from "@sourcebot/schemas/v3/bitbucket.schema"; -import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type"; -import { genericGitHostSchema } from "@sourcebot/schemas/v3/genericGitHost.schema"; -import { gerritSchema } from "@sourcebot/schemas/v3/gerrit.schema"; -import { giteaSchema } from "@sourcebot/schemas/v3/gitea.schema"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; -import { githubSchema } from "@sourcebot/schemas/v3/github.schema"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; -import { gitlabSchema } from "@sourcebot/schemas/v3/gitlab.schema"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; import { getPlan, hasEntitlement } from "@sourcebot/shared"; -import Ajv from "ajv"; import { StatusCodes } from "http-status-codes"; import { cookies, headers } from "next/headers"; import { createTransport } from "nodemailer"; import { Octokit } from "octokit"; import { auth } from "./auth"; -import { getConnection } from "./data/connection"; import { getOrgFromDomain } from "./data/org"; import { decrementOrgSeatCount, getSubscriptionForOrg } from "./ee/features/billing/serverUtils"; import { IS_BILLING_ENABLED } from "./ee/features/billing/stripe"; import InviteUserEmail from "./emails/inviteUserEmail"; import JoinRequestApprovedEmail from "./emails/joinRequestApprovedEmail"; import JoinRequestSubmittedEmail from "./emails/joinRequestSubmittedEmail"; -import { AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME, MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, SEARCH_MODE_COOKIE_NAME, SINGLE_TENANT_ORG_DOMAIN, SOURCEBOT_GUEST_USER_ID, SOURCEBOT_SUPPORT_EMAIL } from "./lib/constants"; +import { AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME, MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, SINGLE_TENANT_ORG_DOMAIN, SOURCEBOT_GUEST_USER_ID, SOURCEBOT_SUPPORT_EMAIL } from "./lib/constants"; import { orgDomainSchema, orgNameSchema, repositoryQuerySchema } from "./lib/schemas"; import { ApiKeyPayload, TenancyMode } from "./lib/types"; -import { withAuthV2, withOptionalAuthV2 } from "./withAuthV2"; - -const ajv = new Ajv({ - validateFormats: false, -}); +import { withOptionalAuthV2 } from "./withAuthV2"; const logger = createLogger('web-actions'); const auditService = getAuditService(); @@ -187,31 +173,6 @@ export const withTenancyModeEnforcement = async(mode: TenancyMode, fn: () => ////// Actions /////// -export const createOrg = async (name: string, domain: string): Promise<{ id: number } | ServiceError> => sew(() => - withTenancyModeEnforcement('multi', () => - withAuth(async (userId) => { - const org = await prisma.org.create({ - data: { - name, - domain, - members: { - create: { - role: "OWNER", - user: { - connect: { - id: userId, - } - } - } - } - } - }); - - return { - id: org.id, - } - }))); - export const updateOrgName = async (name: string, domain: string) => sew(() => withAuth((userId) => withOrgMembership(userId, domain, async ({ org }) => { @@ -573,87 +534,20 @@ export const getUserApiKeys = async (domain: string): Promise<{ name: string; cr })); }))); -export const getConnections = async (domain: string, filter: { status?: ConnectionSyncStatus[] } = {}) => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connections = await prisma.connection.findMany({ - where: { - orgId: org.id, - ...(filter.status ? { - syncStatus: { in: filter.status } - } : {}), - }, - include: { - repos: { - include: { - repo: true, - } - } - } - }); - - return connections.map((connection) => ({ - id: connection.id, - name: connection.name, - syncStatus: connection.syncStatus, - syncStatusMetadata: connection.syncStatusMetadata, - connectionType: connection.connectionType, - updatedAt: connection.updatedAt, - syncedAt: connection.syncedAt ?? undefined, - linkedRepos: connection.repos.map(({ repo }) => ({ - id: repo.id, - name: repo.name, - repoIndexingStatus: repo.repoIndexingStatus, - })), - })); - }) - )); - -export const getConnectionInfo = async (connectionId: number, domain: string) => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connection = await prisma.connection.findUnique({ - where: { - id: connectionId, - orgId: org.id, - }, - include: { - repos: true, - } - }); - - if (!connection) { - return notFound(); - } - - return { - id: connection.id, - name: connection.name, - syncStatus: connection.syncStatus, - syncStatusMetadata: connection.syncStatusMetadata, - connectionType: connection.connectionType, - updatedAt: connection.updatedAt, - syncedAt: connection.syncedAt ?? undefined, - numLinkedRepos: connection.repos.length, - } - }))); - -export const getRepos = async (filter: { status?: RepoIndexingStatus[], connectionId?: number } = {}) => sew(() => +export const getRepos = async ({ + where, + take, +}: { + where?: Prisma.RepoWhereInput, + take?: number +} = {}) => sew(() => withOptionalAuthV2(async ({ org, prisma }) => { const repos = await prisma.repo.findMany({ where: { orgId: org.id, - ...(filter.status ? { - repoIndexingStatus: { in: filter.status } - } : {}), - ...(filter.connectionId ? { - connections: { - some: { - connectionId: filter.connectionId - } - } - } : {}), - } + ...where, + }, + take, }); return repos.map((repo) => repositoryQuerySchema.parse({ @@ -665,10 +559,63 @@ export const getRepos = async (filter: { status?: RepoIndexingStatus[], connecti webUrl: repo.webUrl ?? undefined, imageUrl: repo.imageUrl ?? undefined, indexedAt: repo.indexedAt ?? undefined, - repoIndexingStatus: repo.repoIndexingStatus, })) })); +/** + * Returns a set of aggregated stats about the repos in the org + */ +export const getReposStats = async () => sew(() => + withOptionalAuthV2(async ({ org, prisma }) => { + const [ + // Total number of repos. + numberOfRepos, + // Number of repos with their first time indexing jobs either + // pending or in progress. + numberOfReposWithFirstTimeIndexingJobsInProgress, + // Number of repos that have been indexed at least once. + numberOfReposWithIndex, + ] = await Promise.all([ + prisma.repo.count({ + where: { + orgId: org.id, + } + }), + prisma.repo.count({ + where: { + orgId: org.id, + jobs: { + some: { + type: RepoIndexingJobType.INDEX, + status: { + in: [ + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, + ] + } + }, + }, + indexedAt: null, + } + }), + prisma.repo.count({ + where: { + orgId: org.id, + NOT: { + indexedAt: null, + } + } + }) + ]); + + return { + numberOfRepos, + numberOfReposWithFirstTimeIndexingJobsInProgress, + numberOfReposWithIndex, + }; + }) +) + export const getRepoInfoByName = async (repoName: string) => sew(() => withOptionalAuthV2(async ({ org, prisma }) => { // @note: repo names are represented by their remote url @@ -725,58 +672,9 @@ export const getRepoInfoByName = async (repoName: string) => sew(() => webUrl: repo.webUrl ?? undefined, imageUrl: repo.imageUrl ?? undefined, indexedAt: repo.indexedAt ?? undefined, - repoIndexingStatus: repo.repoIndexingStatus, } })); -export const createConnection = async (name: string, type: CodeHostType, connectionConfig: string, domain: string): Promise<{ id: number } | ServiceError> => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - if (env.CONFIG_PATH !== undefined) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.CONNECTION_CONFIG_PATH_SET, - message: "A configuration file has been provided. New connections cannot be added through the web interface.", - } satisfies ServiceError; - } - - const parsedConfig = parseConnectionConfig(connectionConfig); - if (isServiceError(parsedConfig)) { - return parsedConfig; - } - - const existingConnectionWithName = await prisma.connection.findUnique({ - where: { - name_orgId: { - orgId: org.id, - name, - } - } - }); - - if (existingConnectionWithName) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.CONNECTION_ALREADY_EXISTS, - message: "A connection with this name already exists.", - } satisfies ServiceError; - } - - const connection = await prisma.connection.create({ - data: { - orgId: org.id, - name, - config: parsedConfig as unknown as Prisma.InputJsonValue, - connectionType: type, - } - }); - - return { - id: connection.id, - } - }, OrgRole.OWNER) - )); - export const experimental_addGithubRepositoryByUrl = async (repositoryUrl: string): Promise<{ connectionId: number } | ServiceError> => sew(() => withOptionalAuthV2(async ({ org, prisma }) => { if (env.EXPERIMENT_SELF_SERVE_REPO_INDEXING_ENABLED !== 'true') { @@ -913,148 +811,6 @@ export const experimental_addGithubRepositoryByUrl = async (repositoryUrl: strin } })); -export const updateConnectionDisplayName = async (connectionId: number, name: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connection = await getConnection(connectionId, org.id); - if (!connection) { - return notFound(); - } - - const existingConnectionWithName = await prisma.connection.findUnique({ - where: { - name_orgId: { - orgId: org.id, - name, - } - } - }); - - if (existingConnectionWithName) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.CONNECTION_ALREADY_EXISTS, - message: "A connection with this name already exists.", - } satisfies ServiceError; - } - - await prisma.connection.update({ - where: { - id: connectionId, - orgId: org.id, - }, - data: { - name, - } - }); - - return { - success: true, - } - }, OrgRole.OWNER) - )); - -export const updateConnectionConfigAndScheduleSync = async (connectionId: number, config: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connection = await getConnection(connectionId, org.id); - if (!connection) { - return notFound(); - } - - const parsedConfig = parseConnectionConfig(config); - if (isServiceError(parsedConfig)) { - return parsedConfig; - } - - if (connection.syncStatus === "SYNC_NEEDED" || - connection.syncStatus === "IN_SYNC_QUEUE" || - connection.syncStatus === "SYNCING") { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.CONNECTION_SYNC_ALREADY_SCHEDULED, - message: "Connection is already syncing. Please wait for the sync to complete before updating the connection.", - } satisfies ServiceError; - } - - await prisma.connection.update({ - where: { - id: connectionId, - orgId: org.id, - }, - data: { - config: parsedConfig as unknown as Prisma.InputJsonValue, - syncStatus: "SYNC_NEEDED", - } - }); - - return { - success: true, - } - }, OrgRole.OWNER) - )); - -export const flagConnectionForSync = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connection = await getConnection(connectionId, org.id); - if (!connection || connection.orgId !== org.id) { - return notFound(); - } - - await prisma.connection.update({ - where: { - id: connection.id, - }, - data: { - syncStatus: "SYNC_NEEDED", - } - }); - - return { - success: true, - } - }) - )); - -export const flagReposForIndex = async (repoIds: number[]) => sew(() => - withAuthV2(async ({ org, prisma }) => { - await prisma.repo.updateMany({ - where: { - id: { in: repoIds }, - orgId: org.id, - }, - data: { - repoIndexingStatus: RepoIndexingStatus.NEW, - } - }); - - return { - success: true, - } - })); - -export const deleteConnection = async (connectionId: number, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() => - withAuth((userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const connection = await getConnection(connectionId, org.id); - if (!connection) { - return notFound(); - } - - await prisma.connection.delete({ - where: { - id: connectionId, - orgId: org.id, - } - }); - - return { - success: true, - } - }, OrgRole.OWNER) - )); - export const getCurrentUserRole = async (domain: string): Promise => sew(() => withAuth((userId) => withOrgMembership(userId, domain, async ({ userRole }) => { @@ -1267,13 +1023,6 @@ export const cancelInvite = async (inviteId: string, domain: string): Promise<{ }, /* minRequiredRole = */ OrgRole.OWNER) )); -export const getOrgInviteId = async (domain: string) => sew(() => - withAuth(async (userId) => - withOrgMembership(userId, domain, async ({ org }) => { - return org.inviteLinkId; - }, /* minRequiredRole = */ OrgRole.OWNER) - )); - export const getMe = async () => sew(() => withAuth(async (userId) => { const user = await prisma.user.findUnique({ @@ -1610,27 +1359,6 @@ export const leaveOrg = async (domain: string): Promise<{ success: boolean } | S }) )); - -export const getOrgMembership = async (domain: string) => sew(() => - withAuth(async (userId) => - withOrgMembership(userId, domain, async ({ org }) => { - const membership = await prisma.userToOrg.findUnique({ - where: { - orgId_userId: { - orgId: org.id, - userId: userId, - } - } - }); - - if (!membership) { - return notFound(); - } - - return membership; - }) - )); - export const getOrgMembers = async (domain: string) => sew(() => withAuth(async (userId) => withOrgMembership(userId, domain, async ({ org }) => { @@ -1826,20 +1554,6 @@ export const setMemberApprovalRequired = async (domain: string, required: boolea ) ); -export const getInviteLinkEnabled = async (domain: string): Promise => sew(async () => { - const org = await prisma.org.findUnique({ - where: { - domain, - }, - }); - - if (!org) { - return orgNotFound(); - } - - return org.inviteLinkEnabled; -}); - export const setInviteLinkEnabled = async (domain: string, enabled: boolean): Promise<{ success: boolean } | ServiceError> => sew(async () => withAuth(async (userId) => withOrgMembership(userId, domain, async ({ org }) => { @@ -1971,10 +1685,6 @@ export const rejectAccountRequest = async (requestId: string, domain: string) => }, /* minRequiredRole = */ OrgRole.OWNER) )); -export const dismissMobileUnsupportedSplashScreen = async () => sew(async () => { - await (await cookies()).set(MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, 'true'); - return true; -}); export const getSearchContexts = async (domain: string) => sew(() => withAuth((userId) => @@ -2127,126 +1837,17 @@ export const setAnonymousAccessStatus = async (domain: string, enabled: boolean) }); }); -export async function setSearchModeCookie(searchMode: "precise" | "agentic") { - const cookieStore = await cookies(); - cookieStore.set(SEARCH_MODE_COOKIE_NAME, searchMode, { - httpOnly: false, // Allow client-side access - }); -} - -export async function setAgenticSearchTutorialDismissedCookie(dismissed: boolean) { +export const setAgenticSearchTutorialDismissedCookie = async (dismissed: boolean) => sew(async () => { const cookieStore = await cookies(); cookieStore.set(AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME, dismissed ? "true" : "false", { httpOnly: false, // Allow client-side access + maxAge: 365 * 24 * 60 * 60, // 1 year in seconds }); -} - -////// Helpers /////// - -const parseConnectionConfig = (config: string) => { - let parsedConfig: ConnectionConfig; - try { - parsedConfig = JSON.parse(config); - } catch (_e) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: "config must be a valid JSON object." - } satisfies ServiceError; - } - - const connectionType = parsedConfig.type; - const schema = (() => { - switch (connectionType) { - case "github": - return githubSchema; - case "gitlab": - return gitlabSchema; - case 'gitea': - return giteaSchema; - case 'gerrit': - return gerritSchema; - case 'bitbucket': - return bitbucketSchema; - case 'azuredevops': - return azuredevopsSchema; - case 'git': - return genericGitHostSchema; - } - })(); - - if (!schema) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: "invalid connection type", - } satisfies ServiceError; - } - - const isValidConfig = ajv.validate(schema, parsedConfig); - if (!isValidConfig) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: `config schema validation failed with errors: ${ajv.errorsText(ajv.errors)}`, - } satisfies ServiceError; - } - - if ('token' in parsedConfig && parsedConfig.token && 'env' in parsedConfig.token) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: "Environment variables are not supported for connections created in the web UI. Please use a secret instead.", - } satisfies ServiceError; - } - - const { numRepos, hasToken } = (() => { - switch (connectionType) { - case "gitea": - case "github": - case "bitbucket": - case "azuredevops": { - return { - numRepos: parsedConfig.repos?.length, - hasToken: !!parsedConfig.token, - } - } - case "gitlab": { - return { - numRepos: parsedConfig.projects?.length, - hasToken: !!parsedConfig.token, - } - } - case "gerrit": { - return { - numRepos: parsedConfig.projects?.length, - hasToken: true, // gerrit doesn't use a token atm - } - } - case "git": { - return { - numRepos: 1, - hasToken: false, - } - } - } - })(); - - if (!hasToken && numRepos && numRepos > env.CONFIG_MAX_REPOS_NO_TOKEN) { - return { - statusCode: StatusCodes.BAD_REQUEST, - errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: `You must provide a token to sync more than ${env.CONFIG_MAX_REPOS_NO_TOKEN} repositories.`, - } satisfies ServiceError; - } - - return parsedConfig; -} - -export const encryptValue = async (value: string) => { - return encrypt(value); -} + return true; +}); -export const decryptValue = async (iv: string, encryptedValue: string) => { - return decrypt(iv, encryptedValue); -} \ No newline at end of file +export const dismissMobileUnsupportedSplashScreen = async () => sew(async () => { + const cookieStore = await cookies(); + cookieStore.set(MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME, 'true'); + return true; +}); diff --git a/packages/web/src/app/[domain]/browse/[...path]/components/pureCodePreviewPanel.tsx b/packages/web/src/app/[domain]/browse/[...path]/components/pureCodePreviewPanel.tsx index b2398e9b..49f08be8 100644 --- a/packages/web/src/app/[domain]/browse/[...path]/components/pureCodePreviewPanel.tsx +++ b/packages/web/src/app/[domain]/browse/[...path]/components/pureCodePreviewPanel.tsx @@ -13,7 +13,8 @@ import { search } from "@codemirror/search"; import CodeMirror, { EditorSelection, EditorView, ReactCodeMirrorRef, SelectionRange, ViewUpdate } from "@uiw/react-codemirror"; import { useCallback, useEffect, useMemo, useState } from "react"; import { EditorContextMenu } from "../../../components/editorContextMenu"; -import { BrowseHighlightRange, HIGHLIGHT_RANGE_QUERY_PARAM, useBrowseNavigation } from "../../hooks/useBrowseNavigation"; +import { useBrowseNavigation } from "../../hooks/useBrowseNavigation"; +import { BrowseHighlightRange, HIGHLIGHT_RANGE_QUERY_PARAM } from "../../hooks/utils"; import { useBrowseState } from "../../hooks/useBrowseState"; import { rangeHighlightingExtension } from "./rangeHighlightingExtension"; import useCaptureEvent from "@/hooks/useCaptureEvent"; diff --git a/packages/web/src/app/[domain]/browse/[...path]/components/pureTreePreviewPanel.tsx b/packages/web/src/app/[domain]/browse/[...path]/components/pureTreePreviewPanel.tsx index cdb8f3ca..83c9528e 100644 --- a/packages/web/src/app/[domain]/browse/[...path]/components/pureTreePreviewPanel.tsx +++ b/packages/web/src/app/[domain]/browse/[...path]/components/pureTreePreviewPanel.tsx @@ -3,7 +3,7 @@ import { useRef } from "react"; import { FileTreeItem } from "@/features/fileTree/actions"; import { FileTreeItemComponent } from "@/features/fileTree/components/fileTreeItemComponent"; -import { getBrowsePath } from "../../hooks/useBrowseNavigation"; +import { getBrowsePath } from "../../hooks/utils"; import { ScrollArea } from "@/components/ui/scroll-area"; import { useBrowseParams } from "../../hooks/useBrowseParams"; import { useDomain } from "@/hooks/useDomain"; diff --git a/packages/web/src/app/[domain]/browse/[...path]/components/rangeHighlightingExtension.ts b/packages/web/src/app/[domain]/browse/[...path]/components/rangeHighlightingExtension.ts index cb0fe977..d59ef56a 100644 --- a/packages/web/src/app/[domain]/browse/[...path]/components/rangeHighlightingExtension.ts +++ b/packages/web/src/app/[domain]/browse/[...path]/components/rangeHighlightingExtension.ts @@ -2,7 +2,7 @@ import { StateField, Range } from "@codemirror/state"; import { Decoration, DecorationSet, EditorView } from "@codemirror/view"; -import { BrowseHighlightRange } from "../../hooks/useBrowseNavigation"; +import { BrowseHighlightRange } from "../../hooks/utils"; const markDecoration = Decoration.mark({ class: "searchMatch-selected", diff --git a/packages/web/src/app/[domain]/browse/hooks/useBrowseNavigation.ts b/packages/web/src/app/[domain]/browse/hooks/useBrowseNavigation.ts index 0d79170e..0ab8e71c 100644 --- a/packages/web/src/app/[domain]/browse/hooks/useBrowseNavigation.ts +++ b/packages/web/src/app/[domain]/browse/hooks/useBrowseNavigation.ts @@ -3,58 +3,7 @@ import { useRouter } from "next/navigation"; import { useDomain } from "@/hooks/useDomain"; import { useCallback } from "react"; -import { BrowseState, SET_BROWSE_STATE_QUERY_PARAM } from "../browseStateProvider"; - -export type BrowseHighlightRange = { - start: { lineNumber: number; column: number; }; - end: { lineNumber: number; column: number; }; -} | { - start: { lineNumber: number; }; - end: { lineNumber: number; }; -} - -export const HIGHLIGHT_RANGE_QUERY_PARAM = 'highlightRange'; - -export interface GetBrowsePathProps { - repoName: string; - revisionName?: string; - path: string; - pathType: 'blob' | 'tree'; - highlightRange?: BrowseHighlightRange; - setBrowseState?: Partial; - domain: string; -} - -export const getBrowsePath = ({ - repoName, - revisionName = 'HEAD', - path, - pathType, - highlightRange, - setBrowseState, - domain, -}: GetBrowsePathProps) => { - const params = new URLSearchParams(); - - if (highlightRange) { - const { start, end } = highlightRange; - - if ('column' in start && 'column' in end) { - params.set(HIGHLIGHT_RANGE_QUERY_PARAM, `${start.lineNumber}:${start.column},${end.lineNumber}:${end.column}`); - } else { - params.set(HIGHLIGHT_RANGE_QUERY_PARAM, `${start.lineNumber},${end.lineNumber}`); - } - } - - if (setBrowseState) { - params.set(SET_BROWSE_STATE_QUERY_PARAM, JSON.stringify(setBrowseState)); - } - - const encodedPath = encodeURIComponent(path); - const browsePath = `/${domain}/browse/${repoName}@${revisionName}/-/${pathType}/${encodedPath}${params.size > 0 ? `?${params.toString()}` : ''}`; - return browsePath; -} - +import { getBrowsePath, GetBrowsePathProps } from "./utils"; export const useBrowseNavigation = () => { const router = useRouter(); diff --git a/packages/web/src/app/[domain]/browse/hooks/useBrowsePath.ts b/packages/web/src/app/[domain]/browse/hooks/useBrowsePath.ts index fcf29be8..2165025d 100644 --- a/packages/web/src/app/[domain]/browse/hooks/useBrowsePath.ts +++ b/packages/web/src/app/[domain]/browse/hooks/useBrowsePath.ts @@ -1,7 +1,7 @@ 'use client'; import { useMemo } from "react"; -import { getBrowsePath, GetBrowsePathProps } from "./useBrowseNavigation"; +import { getBrowsePath, GetBrowsePathProps } from "./utils"; import { useDomain } from "@/hooks/useDomain"; export const useBrowsePath = ({ diff --git a/packages/web/src/app/[domain]/browse/hooks/utils.ts b/packages/web/src/app/[domain]/browse/hooks/utils.ts index ba3214fb..804c3864 100644 --- a/packages/web/src/app/[domain]/browse/hooks/utils.ts +++ b/packages/web/src/app/[domain]/browse/hooks/utils.ts @@ -1,3 +1,24 @@ +import { BrowseState, SET_BROWSE_STATE_QUERY_PARAM } from "../browseStateProvider"; + +export const HIGHLIGHT_RANGE_QUERY_PARAM = 'highlightRange'; + +export type BrowseHighlightRange = { + start: { lineNumber: number; column: number; }; + end: { lineNumber: number; column: number; }; +} | { + start: { lineNumber: number; }; + end: { lineNumber: number; }; +} + +export interface GetBrowsePathProps { + repoName: string; + revisionName?: string; + path: string; + pathType: 'blob' | 'tree'; + highlightRange?: BrowseHighlightRange; + setBrowseState?: Partial; + domain: string; +} export const getBrowseParamsFromPathParam = (pathParam: string) => { const sentinelIndex = pathParam.search(/\/-\/(tree|blob)/); @@ -7,7 +28,7 @@ export const getBrowseParamsFromPathParam = (pathParam: string) => { const repoAndRevisionPart = decodeURIComponent(pathParam.substring(0, sentinelIndex)); const lastAtIndex = repoAndRevisionPart.lastIndexOf('@'); - + const repoName = lastAtIndex === -1 ? repoAndRevisionPart : repoAndRevisionPart.substring(0, lastAtIndex); const revisionName = lastAtIndex === -1 ? undefined : repoAndRevisionPart.substring(lastAtIndex + 1); @@ -40,4 +61,29 @@ export const getBrowseParamsFromPathParam = (pathParam: string) => { path, pathType, } -} \ No newline at end of file +}; + +export const getBrowsePath = ({ + repoName, revisionName = 'HEAD', path, pathType, highlightRange, setBrowseState, domain, +}: GetBrowsePathProps) => { + const params = new URLSearchParams(); + + if (highlightRange) { + const { start, end } = highlightRange; + + if ('column' in start && 'column' in end) { + params.set(HIGHLIGHT_RANGE_QUERY_PARAM, `${start.lineNumber}:${start.column},${end.lineNumber}:${end.column}`); + } else { + params.set(HIGHLIGHT_RANGE_QUERY_PARAM, `${start.lineNumber},${end.lineNumber}`); + } + } + + if (setBrowseState) { + params.set(SET_BROWSE_STATE_QUERY_PARAM, JSON.stringify(setBrowseState)); + } + + const encodedPath = encodeURIComponent(path); + const browsePath = `/${domain}/browse/${repoName}@${revisionName}/-/${pathType}/${encodedPath}${params.size > 0 ? `?${params.toString()}` : ''}`; + return browsePath; +}; + diff --git a/packages/web/src/app/[domain]/chat/[id]/page.tsx b/packages/web/src/app/[domain]/chat/[id]/page.tsx index d37076cd..4929589b 100644 --- a/packages/web/src/app/[domain]/chat/[id]/page.tsx +++ b/packages/web/src/app/[domain]/chat/[id]/page.tsx @@ -56,6 +56,7 @@ export default async function Page(props: PageProps) { <>
/ diff --git a/packages/web/src/app/[domain]/components/homepage/askSourcebotDemoCards.tsx b/packages/web/src/app/[domain]/chat/components/demoCards.tsx similarity index 98% rename from packages/web/src/app/[domain]/components/homepage/askSourcebotDemoCards.tsx rename to packages/web/src/app/[domain]/chat/components/demoCards.tsx index 31037607..016d9605 100644 --- a/packages/web/src/app/[domain]/components/homepage/askSourcebotDemoCards.tsx +++ b/packages/web/src/app/[domain]/chat/components/demoCards.tsx @@ -11,13 +11,13 @@ import { cn, getCodeHostIcon } from "@/lib/utils"; import useCaptureEvent from "@/hooks/useCaptureEvent"; import { SearchScopeInfoCard } from "@/features/chat/components/chatBox/searchScopeInfoCard"; -interface AskSourcebotDemoCardsProps { +interface DemoCards { demoExamples: DemoExamples; } -export const AskSourcebotDemoCards = ({ +export const DemoCards = ({ demoExamples, -}: AskSourcebotDemoCardsProps) => { +}: DemoCards) => { const captureEvent = useCaptureEvent(); const [selectedFilterSearchScope, setSelectedFilterSearchScope] = useState(null); diff --git a/packages/web/src/app/[domain]/components/homepage/agenticSearch.tsx b/packages/web/src/app/[domain]/chat/components/landingPageChatBox.tsx similarity index 56% rename from packages/web/src/app/[domain]/components/homepage/agenticSearch.tsx rename to packages/web/src/app/[domain]/chat/components/landingPageChatBox.tsx index 327cd297..7de6b928 100644 --- a/packages/web/src/app/[domain]/components/homepage/agenticSearch.tsx +++ b/packages/web/src/app/[domain]/chat/components/landingPageChatBox.tsx @@ -6,50 +6,32 @@ import { ChatBoxToolbar } from "@/features/chat/components/chatBox/chatBoxToolba import { LanguageModelInfo, SearchScope } from "@/features/chat/types"; import { useCreateNewChatThread } from "@/features/chat/useCreateNewChatThread"; import { RepositoryQuery, SearchContextQuery } from "@/lib/types"; -import { useCallback, useState } from "react"; -import { SearchModeSelector, SearchModeSelectorProps } from "./toolbar"; +import { useState } from "react"; import { useLocalStorage } from "usehooks-ts"; -import { DemoExamples } from "@/types"; -import { AskSourcebotDemoCards } from "./askSourcebotDemoCards"; -import { AgenticSearchTutorialDialog } from "./agenticSearchTutorialDialog"; -import { setAgenticSearchTutorialDismissedCookie } from "@/actions"; -import { RepositorySnapshot } from "./repositorySnapshot"; +import { SearchModeSelector } from "../../components/searchModeSelector"; +import { NotConfiguredErrorBanner } from "@/features/chat/components/notConfiguredErrorBanner"; -interface AgenticSearchProps { - searchModeSelectorProps: SearchModeSelectorProps; +interface LandingPageChatBox { languageModels: LanguageModelInfo[]; repos: RepositoryQuery[]; searchContexts: SearchContextQuery[]; - chatHistory: { - id: string; - createdAt: Date; - name: string | null; - }[]; - demoExamples: DemoExamples | undefined; - isTutorialDismissed: boolean; } -export const AgenticSearch = ({ - searchModeSelectorProps, +export const LandingPageChatBox = ({ languageModels, repos, searchContexts, - demoExamples, - isTutorialDismissed, -}: AgenticSearchProps) => { +}: LandingPageChatBox) => { const { createNewChatThread, isLoading } = useCreateNewChatThread(); const [selectedSearchScopes, setSelectedSearchScopes] = useLocalStorage("selectedSearchScopes", [], { initializeWithValue: false }); const [isContextSelectorOpen, setIsContextSelectorOpen] = useState(false); - - const [isTutorialOpen, setIsTutorialOpen] = useState(!isTutorialDismissed); - const onTutorialDismissed = useCallback(() => { - setIsTutorialOpen(false); - setAgenticSearchTutorialDismissedCookie(true); - }, []); + const isChatBoxDisabled = languageModels.length === 0; return ( -
-
+
+ + +
{ createNewChatThread(children, selectedSearchScopes); @@ -60,6 +42,7 @@ export const AgenticSearch = ({ selectedSearchScopes={selectedSearchScopes} searchContexts={searchContexts} onContextSelectorOpenChanged={setIsContextSelectorOpen} + isDisabled={isChatBoxDisabled} />
@@ -74,33 +57,15 @@ export const AgenticSearch = ({ onContextSelectorOpenChanged={setIsContextSelectorOpen} />
-
- -
- -
- -
- - {demoExamples && ( - - )} - - {isTutorialOpen && ( - + {isChatBoxDisabled && ( + )}
) diff --git a/packages/web/src/app/[domain]/chat/components/newChatPanel.tsx b/packages/web/src/app/[domain]/chat/components/newChatPanel.tsx deleted file mode 100644 index 91ae3f96..00000000 --- a/packages/web/src/app/[domain]/chat/components/newChatPanel.tsx +++ /dev/null @@ -1,72 +0,0 @@ -'use client'; - -import { ResizablePanel } from "@/components/ui/resizable"; -import { ChatBox } from "@/features/chat/components/chatBox"; -import { ChatBoxToolbar } from "@/features/chat/components/chatBox/chatBoxToolbar"; -import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; -import { useCreateNewChatThread } from "@/features/chat/useCreateNewChatThread"; -import { LanguageModelInfo, SearchScope } from "@/features/chat/types"; -import { RepositoryQuery, SearchContextQuery } from "@/lib/types"; -import { useCallback, useState } from "react"; -import { Descendant } from "slate"; -import { useLocalStorage } from "usehooks-ts"; - -interface NewChatPanelProps { - languageModels: LanguageModelInfo[]; - repos: RepositoryQuery[]; - searchContexts: SearchContextQuery[]; - order: number; -} - -export const NewChatPanel = ({ - languageModels, - repos, - searchContexts, - order, -}: NewChatPanelProps) => { - const [selectedSearchScopes, setSelectedSearchScopes] = useLocalStorage("selectedSearchScopes", [], { initializeWithValue: false }); - const { createNewChatThread, isLoading } = useCreateNewChatThread(); - const [isContextSelectorOpen, setIsContextSelectorOpen] = useState(false); - - const onSubmit = useCallback((children: Descendant[]) => { - createNewChatThread(children, selectedSearchScopes); - }, [createNewChatThread, selectedSearchScopes]); - - - return ( - -
-

What can I help you understand?

-
- - -
- -
- -
-
- - ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/homepage/agenticSearchTutorialDialog.tsx b/packages/web/src/app/[domain]/chat/components/tutorialDialog.tsx similarity index 95% rename from packages/web/src/app/[domain]/components/homepage/agenticSearchTutorialDialog.tsx rename to packages/web/src/app/[domain]/chat/components/tutorialDialog.tsx index a44d6735..c5ddd85d 100644 --- a/packages/web/src/app/[domain]/components/homepage/agenticSearchTutorialDialog.tsx +++ b/packages/web/src/app/[domain]/chat/components/tutorialDialog.tsx @@ -1,7 +1,8 @@ "use client" +import { setAgenticSearchTutorialDismissedCookie } from "@/actions" import { Button } from "@/components/ui/button" -import { Dialog, DialogContent } from "@/components/ui/dialog" +import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog" import { ModelProviderLogo } from "@/features/chat/components/chatBox/modelProviderLogo" import { cn } from "@/lib/utils" import mentionsDemo from "@/public/ask_sb_tutorial_at_mentions.png" @@ -27,11 +28,9 @@ import { } from "lucide-react" import Image from "next/image" import Link from "next/link" -import { useState } from "react" +import { useCallback, useState } from "react" + -interface AgenticSearchTutorialDialogProps { - onClose: () => void -} // Star button component that fetches GitHub star count @@ -249,7 +248,17 @@ const tutorialSteps = [ }, ] -export const AgenticSearchTutorialDialog = ({ onClose }: AgenticSearchTutorialDialogProps) => { +interface TutorialDialogProps { + isOpen: boolean; +} + +export const TutorialDialog = ({ isOpen: _isOpen }: TutorialDialogProps) => { + const [isOpen, setIsOpen] = useState(_isOpen); + const onClose = useCallback(() => { + setIsOpen(false); + setAgenticSearchTutorialDismissedCookie(true); + }, []); + const [currentStep, setCurrentStep] = useState(0) const nextStep = () => { @@ -269,11 +278,12 @@ export const AgenticSearchTutorialDialog = ({ onClose }: AgenticSearchTutorialDi const currentStepData = tutorialSteps[currentStep]; return ( - + + Ask Sourcebot tutorial
{/* Left Column (Text Content & Navigation) */}
diff --git a/packages/web/src/app/[domain]/chat/layout.tsx b/packages/web/src/app/[domain]/chat/layout.tsx index e82ea91f..2968c748 100644 --- a/packages/web/src/app/[domain]/chat/layout.tsx +++ b/packages/web/src/app/[domain]/chat/layout.tsx @@ -1,10 +1,14 @@ +import { AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME } from '@/lib/constants'; import { NavigationGuardProvider } from 'next-navigation-guard'; +import { cookies } from 'next/headers'; +import { TutorialDialog } from './components/tutorialDialog'; interface LayoutProps { children: React.ReactNode; } export default async function Layout({ children }: LayoutProps) { + const isTutorialDismissed = (await cookies()).get(AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME)?.value === "true"; return ( // @note: we use a navigation guard here since we don't support resuming streams yet. @@ -13,6 +17,7 @@ export default async function Layout({ children }: LayoutProps) {
{children}
+ ) } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/chat/page.tsx b/packages/web/src/app/[domain]/chat/page.tsx index 8bdddf9e..5b7afef2 100644 --- a/packages/web/src/app/[domain]/chat/page.tsx +++ b/packages/web/src/app/[domain]/chat/page.tsx @@ -1,13 +1,17 @@ -import { getRepos, getSearchContexts } from "@/actions"; -import { getUserChatHistory, getConfiguredLanguageModelsInfo } from "@/features/chat/actions"; +import { getRepos, getReposStats, getSearchContexts } from "@/actions"; +import { SourcebotLogo } from "@/app/components/sourcebotLogo"; +import { getConfiguredLanguageModelsInfo } from "@/features/chat/actions"; +import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; import { ServiceErrorException } from "@/lib/serviceError"; -import { isServiceError } from "@/lib/utils"; -import { NewChatPanel } from "./components/newChatPanel"; -import { TopBar } from "../components/topBar"; -import { ResizablePanelGroup } from "@/components/ui/resizable"; -import { ChatSidePanel } from "./components/chatSidePanel"; -import { auth } from "@/auth"; -import { AnimatedResizableHandle } from "@/components/ui/animatedResizableHandle"; +import { isServiceError, measure } from "@/lib/utils"; +import { LandingPageChatBox } from "./components/landingPageChatBox"; +import { RepositoryCarousel } from "../components/repositoryCarousel"; +import { NavigationMenu } from "../components/navigationMenu"; +import { Separator } from "@/components/ui/separator"; +import { DemoCards } from "./components/demoCards"; +import { env } from "@/env.mjs"; +import { loadJsonFile } from "@sourcebot/shared"; +import { DemoExamples, demoExamplesSchema } from "@/types"; interface PageProps { params: Promise<{ @@ -18,47 +22,85 @@ interface PageProps { export default async function Page(props: PageProps) { const params = await props.params; const languageModels = await getConfiguredLanguageModelsInfo(); - const repos = await getRepos(); const searchContexts = await getSearchContexts(params.domain); - const session = await auth(); - const chatHistory = session ? await getUserChatHistory(params.domain) : []; + const allRepos = await getRepos(); - if (isServiceError(chatHistory)) { - throw new ServiceErrorException(chatHistory); - } + const carouselRepos = await getRepos({ + where: { + indexedAt: { + not: null, + }, + }, + take: 10, + }); + + const repoStats = await getReposStats(); - if (isServiceError(repos)) { - throw new ServiceErrorException(repos); + if (isServiceError(allRepos)) { + throw new ServiceErrorException(allRepos); } if (isServiceError(searchContexts)) { throw new ServiceErrorException(searchContexts); } - const indexedRepos = repos.filter((repo) => repo.indexedAt !== undefined); + if (isServiceError(carouselRepos)) { + throw new ServiceErrorException(carouselRepos); + } + + if (isServiceError(repoStats)) { + throw new ServiceErrorException(repoStats); + } + + const demoExamples = env.SOURCEBOT_DEMO_EXAMPLES_PATH ? await (async () => { + try { + return (await measure(() => loadJsonFile(env.SOURCEBOT_DEMO_EXAMPLES_PATH!, demoExamplesSchema), 'loadExamplesJsonFile')).data; + } catch (error) { + console.error('Failed to load demo examples:', error); + return undefined; + } + })() : undefined; return ( - <> - + - - - - - - + +
+
+ +
+ + + + +
+ +
+ + {demoExamples && ( + <> +
+ +
+ + + + )} + +
+
) } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/editorContextMenu.tsx b/packages/web/src/app/[domain]/components/editorContextMenu.tsx index 102f9f89..091dba7e 100644 --- a/packages/web/src/app/[domain]/components/editorContextMenu.tsx +++ b/packages/web/src/app/[domain]/components/editorContextMenu.tsx @@ -9,7 +9,7 @@ import { Link2Icon } from "@radix-ui/react-icons"; import { EditorView, SelectionRange } from "@uiw/react-codemirror"; import { useCallback, useEffect, useRef } from "react"; import { useDomain } from "@/hooks/useDomain"; -import { HIGHLIGHT_RANGE_QUERY_PARAM } from "@/app/[domain]/browse/hooks/useBrowseNavigation"; +import { HIGHLIGHT_RANGE_QUERY_PARAM } from "../browse/hooks/utils"; interface ContextMenuProps { view: EditorView; diff --git a/packages/web/src/app/[domain]/components/errorNavIndicator.tsx b/packages/web/src/app/[domain]/components/errorNavIndicator.tsx deleted file mode 100644 index 4f7810aa..00000000 --- a/packages/web/src/app/[domain]/components/errorNavIndicator.tsx +++ /dev/null @@ -1,137 +0,0 @@ -"use client"; - -import Link from "next/link"; -import { HoverCard, HoverCardTrigger, HoverCardContent } from "@/components/ui/hover-card"; -import { CircleXIcon } from "lucide-react"; -import { useDomain } from "@/hooks/useDomain"; -import { unwrapServiceError } from "@/lib/utils"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { env } from "@/env.mjs"; -import { useQuery } from "@tanstack/react-query"; -import { ConnectionSyncStatus, RepoIndexingStatus } from "@sourcebot/db"; -import { getConnections } from "@/actions"; -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; -import { getRepos } from "@/app/api/(client)/client"; - -export const ErrorNavIndicator = () => { - const domain = useDomain(); - const captureEvent = useCaptureEvent(); - - const { data: repos, isPending: isPendingRepos, isError: isErrorRepos } = useQuery({ - queryKey: ['repos', domain], - queryFn: () => unwrapServiceError(getRepos()), - select: (data) => data.filter(repo => repo.repoIndexingStatus === RepoIndexingStatus.FAILED), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - const { data: connections, isPending: isPendingConnections, isError: isErrorConnections } = useQuery({ - queryKey: ['connections', domain], - queryFn: () => unwrapServiceError(getConnections(domain)), - select: (data) => data.filter(connection => connection.syncStatus === ConnectionSyncStatus.FAILED), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - if (isPendingRepos || isErrorRepos || isPendingConnections || isErrorConnections) { - return null; - } - - if (repos.length === 0 && connections.length === 0) { - return null; - } - - return ( - - captureEvent('wa_error_nav_hover', {})}> - captureEvent('wa_error_nav_pressed', {})}> -
- - {repos.length + connections.length > 0 && ( - {repos.length + connections.length} - )} -
- - - -
- {connections.length > 0 && ( -
-
-
-

Connection Sync Issues

-
-

- The following connections have failed to sync: -

-
- - {connections - .slice(0, 10) - .map(connection => ( - captureEvent('wa_error_nav_job_pressed', {})}> -
- - - {connection.name} - - - {connection.name} - - -
- - ))} - - {connections.length > 10 && ( -
- And {connections.length - 10} more... -
- )} -
-
- )} - - {repos.length > 0 && ( -
-
-
-

Repository Indexing Issues

-
-

- The following repositories failed to index: -

-
- - {repos - .slice(0, 10) - .map(repo => ( -
- - - {repo.repoName} - - - {repo.repoName} - - -
- ))} - - {repos.length > 10 && ( -
- And {repos.length - 10} more... -
- )} -
-
- )} -
- - - ); -}; diff --git a/packages/web/src/app/[domain]/components/homepage/index.tsx b/packages/web/src/app/[domain]/components/homepage/index.tsx deleted file mode 100644 index fa230288..00000000 --- a/packages/web/src/app/[domain]/components/homepage/index.tsx +++ /dev/null @@ -1,102 +0,0 @@ -'use client'; - -import { SourcebotLogo } from "@/app/components/sourcebotLogo"; -import { LanguageModelInfo } from "@/features/chat/types"; -import { RepositoryQuery, SearchContextQuery } from "@/lib/types"; -import { useHotkeys } from "react-hotkeys-hook"; -import { AgenticSearch } from "./agenticSearch"; -import { PreciseSearch } from "./preciseSearch"; -import { SearchMode } from "./toolbar"; -import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; -import { setSearchModeCookie } from "@/actions"; -import { useCallback, useState } from "react"; -import { DemoExamples } from "@/types"; - -interface HomepageProps { - initialRepos: RepositoryQuery[]; - searchContexts: SearchContextQuery[]; - languageModels: LanguageModelInfo[]; - chatHistory: { - id: string; - createdAt: Date; - name: string | null; - }[]; - initialSearchMode: SearchMode; - demoExamples: DemoExamples | undefined; - isAgenticSearchTutorialDismissed: boolean; -} - - -export const Homepage = ({ - initialRepos, - searchContexts, - languageModels, - chatHistory, - initialSearchMode, - demoExamples, - isAgenticSearchTutorialDismissed, -}: HomepageProps) => { - const [searchMode, setSearchMode] = useState(initialSearchMode); - const isAgenticSearchEnabled = languageModels.length > 0; - - const onSearchModeChanged = useCallback(async (newMode: SearchMode) => { - setSearchMode(newMode); - await setSearchModeCookie(newMode); - }, [setSearchMode]); - - useHotkeys("mod+i", (e) => { - e.preventDefault(); - onSearchModeChanged("agentic"); - }, { - enableOnFormTags: true, - enableOnContentEditable: true, - description: "Switch to agentic search", - }); - - useHotkeys("mod+p", (e) => { - e.preventDefault(); - onSearchModeChanged("precise"); - }, { - enableOnFormTags: true, - enableOnContentEditable: true, - description: "Switch to precise search", - }); - - return ( -
-
- -
- - {searchMode === "precise" ? ( - - ) : ( - - - - )} -
- ) -} - diff --git a/packages/web/src/app/[domain]/components/homepage/preciseSearch.tsx b/packages/web/src/app/[domain]/components/homepage/preciseSearch.tsx deleted file mode 100644 index d5608940..00000000 --- a/packages/web/src/app/[domain]/components/homepage/preciseSearch.tsx +++ /dev/null @@ -1,145 +0,0 @@ -'use client'; - -import { Separator } from "@/components/ui/separator"; -import { SyntaxReferenceGuideHint } from "../syntaxReferenceGuideHint"; -import { RepositorySnapshot } from "./repositorySnapshot"; -import { RepositoryQuery } from "@/lib/types"; -import { useDomain } from "@/hooks/useDomain"; -import Link from "next/link"; -import { SearchBar } from "../searchBar/searchBar"; -import { SearchModeSelector, SearchModeSelectorProps } from "./toolbar"; - -interface PreciseSearchProps { - initialRepos: RepositoryQuery[]; - searchModeSelectorProps: SearchModeSelectorProps; -} - -export const PreciseSearch = ({ - initialRepos, - searchModeSelectorProps, -}: PreciseSearchProps) => { - const domain = useDomain(); - - return ( - <> -
- - -
- -
-
-
- -
-
- - How to search -
- - - test todo (both test and todo) - - - test or todo (either test or todo) - - - {`"exit boot"`} (exact match) - - - TODO case:yes (case sensitive) - - - - - file:README setup (by filename) - - - repo:torvalds/linux test (by repo) - - - lang:typescript (by language) - - - rev:HEAD (by branch or tag) - - - - - file:{`\\.py$`} {`(files that end in ".py")`} - - - sym:main {`(symbols named "main")`} - - - todo -lang:c (negate filter) - - - content:README (search content only) - - -
- -
- - ) -} - -const HowToSection = ({ title, children }: { title: string, children: React.ReactNode }) => { - return ( -
- {title} - {children} -
- ) - -} - -const Highlight = ({ children }: { children: React.ReactNode }) => { - return ( - - {children} - - ) -} - -const QueryExample = ({ children }: { children: React.ReactNode }) => { - return ( - - {children} - - ) -} - -const QueryExplanation = ({ children }: { children: React.ReactNode }) => { - return ( - - {children} - - ) -} - -const Query = ({ query, domain, children }: { query: string, domain: string, children: React.ReactNode }) => { - return ( - - {children} - - ) -} diff --git a/packages/web/src/app/[domain]/components/homepage/repositoryCarousel.tsx b/packages/web/src/app/[domain]/components/homepage/repositoryCarousel.tsx deleted file mode 100644 index 0a90e1a7..00000000 --- a/packages/web/src/app/[domain]/components/homepage/repositoryCarousel.tsx +++ /dev/null @@ -1,105 +0,0 @@ -'use client'; - -import { - Carousel, - CarouselContent, - CarouselItem, -} from "@/components/ui/carousel"; -import Autoscroll from "embla-carousel-auto-scroll"; -import { getCodeHostInfoForRepo } from "@/lib/utils"; -import Image from "next/image"; -import { FileIcon } from "@radix-ui/react-icons"; -import clsx from "clsx"; -import { RepositoryQuery } from "@/lib/types"; -import { getBrowsePath } from "../../browse/hooks/useBrowseNavigation"; -import Link from "next/link"; -import { useDomain } from "@/hooks/useDomain"; - -interface RepositoryCarouselProps { - repos: RepositoryQuery[]; -} - -export const RepositoryCarousel = ({ - repos, -}: RepositoryCarouselProps) => { - return ( - - - {repos.map((repo, index) => ( - - - - ))} - - - ) -}; - -interface RepositoryBadgeProps { - repo: RepositoryQuery; -} - -const RepositoryBadge = ({ - repo -}: RepositoryBadgeProps) => { - const domain = useDomain(); - const { repoIcon, displayName } = (() => { - const info = getCodeHostInfoForRepo({ - codeHostType: repo.codeHostType, - name: repo.repoName, - displayName: repo.repoDisplayName, - webUrl: repo.webUrl, - }); - - if (info) { - return { - repoIcon: {info.codeHostName}, - displayName: info.displayName, - } - } - - return { - repoIcon: , - displayName: repo.repoName, - } - })(); - - return ( - - {repoIcon} - - {displayName} - - - ) -} diff --git a/packages/web/src/app/[domain]/components/homepage/repositorySnapshot.tsx b/packages/web/src/app/[domain]/components/homepage/repositorySnapshot.tsx deleted file mode 100644 index d6845fa2..00000000 --- a/packages/web/src/app/[domain]/components/homepage/repositorySnapshot.tsx +++ /dev/null @@ -1,156 +0,0 @@ -"use client"; - -import Link from "next/link"; -import { RepositoryCarousel } from "./repositoryCarousel"; -import { useDomain } from "@/hooks/useDomain"; -import { useQuery } from "@tanstack/react-query"; -import { unwrapServiceError } from "@/lib/utils"; -import { getRepos } from "@/app/api/(client)/client"; -import { env } from "@/env.mjs"; -import { Skeleton } from "@/components/ui/skeleton"; -import { - Carousel, - CarouselContent, - CarouselItem, -} from "@/components/ui/carousel"; -import { RepoIndexingStatus } from "@sourcebot/db"; -import { SymbolIcon } from "@radix-ui/react-icons"; -import { RepositoryQuery } from "@/lib/types"; -import { captureEvent } from "@/hooks/useCaptureEvent"; - -interface RepositorySnapshotProps { - repos: RepositoryQuery[]; -} - -const MAX_REPOS_TO_DISPLAY_IN_CAROUSEL = 15; - -export function RepositorySnapshot({ - repos: initialRepos, -}: RepositorySnapshotProps) { - const domain = useDomain(); - - const { data: repos, isPending, isError } = useQuery({ - queryKey: ['repos', domain], - queryFn: () => unwrapServiceError(getRepos()), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - placeholderData: initialRepos, - }); - - if (isPending || isError || !repos) { - return ( -
- -
- ) - } - - // Use `indexedAt` to determine if a repo has __ever__ been indexed. - // The repo indexing status only tells us the repo's current indexing status. - const indexedRepos = repos.filter((repo) => repo.indexedAt !== undefined); - - // If there are no indexed repos... - if (indexedRepos.length === 0) { - - // ... show a loading state if repos are being indexed now - if (repos.some((repo) => repo.repoIndexingStatus === RepoIndexingStatus.INDEXING || repo.repoIndexingStatus === RepoIndexingStatus.IN_INDEX_QUEUE)) { - return ( -
- - indexing in progress... -
- ) - - // ... otherwise, show the empty state. - } else { - return ( - - ) - } - } - - return ( -
- - {`${indexedRepos.length} `} - - {indexedRepos.length > 1 ? 'repositories' : 'repository'} - - {` indexed`} - - - {process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === "demo" && ( -

- Interested in using Sourcebot on your code? Check out our{' '} - captureEvent('wa_demo_docs_link_pressed', {})} - > - docs - -

- )} -
- ) -} - -function EmptyRepoState() { - return ( -
- No repositories found - -
-
- - <> - Create a{" "} - - connection - {" "} - to start indexing repositories - - -
-
-
- ) -} - -function RepoSkeleton() { - return ( -
- {/* Skeleton for "Search X repositories" text */} -
- {/* "Search X" */} - {/* "repositories" */} -
- - {/* Skeleton for repository carousel */} - - - {[1, 2, 3].map((_, index) => ( - -
- {/* Icon */} - {/* Repository name */} -
- - ))} - - -
- ) -} diff --git a/packages/web/src/app/[domain]/components/navigationMenu.tsx b/packages/web/src/app/[domain]/components/navigationMenu.tsx deleted file mode 100644 index b71b7ab7..00000000 --- a/packages/web/src/app/[domain]/components/navigationMenu.tsx +++ /dev/null @@ -1,151 +0,0 @@ -import { Button } from "@/components/ui/button"; -import { NavigationMenu as NavigationMenuBase, NavigationMenuItem, NavigationMenuLink, NavigationMenuList, navigationMenuTriggerStyle } from "@/components/ui/navigation-menu"; -import Link from "next/link"; -import { Separator } from "@/components/ui/separator"; -import { SettingsDropdown } from "./settingsDropdown"; -import { GitHubLogoIcon, DiscordLogoIcon } from "@radix-ui/react-icons"; -import { redirect } from "next/navigation"; -import { OrgSelector } from "./orgSelector"; -import { ErrorNavIndicator } from "./errorNavIndicator"; -import { WarningNavIndicator } from "./warningNavIndicator"; -import { ProgressNavIndicator } from "./progressNavIndicator"; -import { SourcebotLogo } from "@/app/components/sourcebotLogo"; -import { TrialNavIndicator } from "./trialNavIndicator"; -import { IS_BILLING_ENABLED } from "@/ee/features/billing/stripe"; -import { env } from "@/env.mjs"; -import { getSubscriptionInfo } from "@/ee/features/billing/actions"; -import { auth } from "@/auth"; -import WhatsNewIndicator from "./whatsNewIndicator"; - -const SOURCEBOT_DISCORD_URL = "https://discord.gg/6Fhp27x7Pb"; -const SOURCEBOT_GITHUB_URL = "https://github.com/sourcebot-dev/sourcebot"; - -interface NavigationMenuProps { - domain: string; -} - -export const NavigationMenu = async ({ - domain, -}: NavigationMenuProps) => { - const subscription = IS_BILLING_ENABLED ? await getSubscriptionInfo(domain) : null; - const session = await auth(); - const isAuthenticated = session?.user !== undefined; - - return ( -
-
-
- - - - - {env.SOURCEBOT_TENANCY_MODE === 'multi' && ( - <> - - - - )} - - - - - - Search - - - - - Repositories - - - {isAuthenticated && ( - <> - {env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === undefined && ( - - - Agents - - - )} - - - Connections - - - - - Settings - - - - )} - - -
- -
- - - - - -
{ - "use server"; - redirect(SOURCEBOT_DISCORD_URL); - }} - > - -
-
{ - "use server"; - redirect(SOURCEBOT_GITHUB_URL); - }} - > - -
- -
-
- -
- - - ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/navigationMenu/index.tsx b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx new file mode 100644 index 00000000..54842731 --- /dev/null +++ b/packages/web/src/app/[domain]/components/navigationMenu/index.tsx @@ -0,0 +1,143 @@ +import { getRepos, getReposStats } from "@/actions"; +import { SourcebotLogo } from "@/app/components/sourcebotLogo"; +import { auth } from "@/auth"; +import { Button } from "@/components/ui/button"; +import { NavigationMenu as NavigationMenuBase } from "@/components/ui/navigation-menu"; +import { Separator } from "@/components/ui/separator"; +import { getSubscriptionInfo } from "@/ee/features/billing/actions"; +import { IS_BILLING_ENABLED } from "@/ee/features/billing/stripe"; +import { env } from "@/env.mjs"; +import { ServiceErrorException } from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { DiscordLogoIcon, GitHubLogoIcon } from "@radix-ui/react-icons"; +import { RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import Link from "next/link"; +import { redirect } from "next/navigation"; +import { OrgSelector } from "../orgSelector"; +import { SettingsDropdown } from "../settingsDropdown"; +import WhatsNewIndicator from "../whatsNewIndicator"; +import { NavigationItems } from "./navigationItems"; +import { ProgressIndicator } from "./progressIndicator"; +import { TrialIndicator } from "./trialIndicator"; + +const SOURCEBOT_DISCORD_URL = "https://discord.gg/6Fhp27x7Pb"; +const SOURCEBOT_GITHUB_URL = "https://github.com/sourcebot-dev/sourcebot"; + +interface NavigationMenuProps { + domain: string; +} + +export const NavigationMenu = async ({ + domain, +}: NavigationMenuProps) => { + const subscription = IS_BILLING_ENABLED ? await getSubscriptionInfo(domain) : null; + const session = await auth(); + const isAuthenticated = session?.user !== undefined; + + const repoStats = await getReposStats(); + if (isServiceError(repoStats)) { + throw new ServiceErrorException(repoStats); + } + + const sampleRepos = await getRepos({ + where: { + jobs: { + some: { + type: RepoIndexingJobType.INDEX, + status: { + in: [ + RepoIndexingJobStatus.PENDING, + RepoIndexingJobStatus.IN_PROGRESS, + ] + } + }, + }, + indexedAt: null, + }, + take: 5, + }); + + if (isServiceError(sampleRepos)) { + throw new ServiceErrorException(sampleRepos); + } + + const { + numberOfRepos, + numberOfReposWithFirstTimeIndexingJobsInProgress, + } = repoStats; + + return ( +
+
+
+ + + + + {env.SOURCEBOT_TENANCY_MODE === 'multi' && ( + <> + + + + )} + + + + +
+ +
+ + + +
{ + "use server"; + redirect(SOURCEBOT_DISCORD_URL); + }} + > + +
+
{ + "use server"; + redirect(SOURCEBOT_GITHUB_URL); + }} + > + +
+ +
+
+ +
+ ) +} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx b/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx new file mode 100644 index 00000000..52db3315 --- /dev/null +++ b/packages/web/src/app/[domain]/components/navigationMenu/navigationItems.tsx @@ -0,0 +1,89 @@ +"use client"; + +import { NavigationMenuItem, NavigationMenuLink, NavigationMenuList, navigationMenuTriggerStyle } from "@/components/ui/navigation-menu"; +import { Badge } from "@/components/ui/badge"; +import { cn, getShortenedNumberDisplayString } from "@/lib/utils"; +import { SearchIcon, MessageCircleIcon, BookMarkedIcon, SettingsIcon, CircleIcon } from "lucide-react"; +import { usePathname } from "next/navigation"; + +interface NavigationItemsProps { + domain: string; + numberOfRepos: number; + numberOfReposWithFirstTimeIndexingJobsInProgress: number; + isAuthenticated: boolean; +} + +export const NavigationItems = ({ + domain, + numberOfRepos, + numberOfReposWithFirstTimeIndexingJobsInProgress, + isAuthenticated, +}: NavigationItemsProps) => { + const pathname = usePathname(); + + const isActive = (href: string) => { + if (href === `/${domain}`) { + return pathname === `/${domain}`; + } + return pathname.startsWith(href); + }; + + return ( + + + + + Search + + {((isActive(`/${domain}`) || isActive(`/${domain}/search`)) && )} + + + + + Ask + + {isActive(`/${domain}/chat`) && } + + + + + Repositories + + {getShortenedNumberDisplayString(numberOfRepos)} + {numberOfReposWithFirstTimeIndexingJobsInProgress > 0 && ( + + )} + + + {isActive(`/${domain}/repos`) && } + + {isAuthenticated && ( + + + + Settings + + {isActive(`/${domain}/settings`) && } + + )} + + ); +}; + +const ActiveIndicator = () => { + return ( +
+ ); +}; \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx b/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx new file mode 100644 index 00000000..8e1889df --- /dev/null +++ b/packages/web/src/app/[domain]/components/navigationMenu/progressIndicator.tsx @@ -0,0 +1,122 @@ +'use client'; + +import { useToast } from "@/components/hooks/use-toast"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Separator } from "@/components/ui/separator"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { useDomain } from "@/hooks/useDomain"; +import { RepositoryQuery } from "@/lib/types"; +import { getCodeHostInfoForRepo, getShortenedNumberDisplayString } from "@/lib/utils"; +import clsx from "clsx"; +import { FileIcon, Loader2Icon, RefreshCwIcon } from "lucide-react"; +import Image from "next/image"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useMemo } from "react"; + +interface ProgressIndicatorProps { + numberOfReposWithFirstTimeIndexingJobsInProgress: number; + sampleRepos: RepositoryQuery[]; +} + +export const ProgressIndicator = ({ + numberOfReposWithFirstTimeIndexingJobsInProgress: numRepos, + sampleRepos, +}: ProgressIndicatorProps) => { + const domain = useDomain(); + const router = useRouter(); + const { toast } = useToast(); + + if (numRepos === 0) { + return null; + } + + const numReposString = getShortenedNumberDisplayString(numRepos); + + return ( + + + + + + {numReposString} + + + + +
+

{`Syncing ${numReposString} ${numRepos === 1 ? 'repository' : 'repositories'}`}

+ +
+ +
+ {sampleRepos.map((repo) => ( + + ))} +
+ {numRepos > sampleRepos.length && ( +
+ + {`View ${numRepos - sampleRepos.length} more`} + +
+ )} + + + ) +} + +const RepoItem = ({ repo }: { repo: RepositoryQuery }) => { + + const { repoIcon, displayName } = useMemo(() => { + const info = getCodeHostInfoForRepo({ + name: repo.repoName, + codeHostType: repo.codeHostType, + displayName: repo.repoDisplayName, + webUrl: repo.webUrl, + }); + + if (info) { + return { + repoIcon: {info.codeHostName}, + displayName: info.displayName, + } + } + + return { + repoIcon: , + displayName: repo.repoName, + } + + + }, [repo.repoName, repo.codeHostType, repo.repoDisplayName, repo.webUrl]); + + + return ( +
+ {repoIcon} + + {displayName} + +
+ ) +} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/trialNavIndicator.tsx b/packages/web/src/app/[domain]/components/navigationMenu/trialIndicator.tsx similarity index 95% rename from packages/web/src/app/[domain]/components/trialNavIndicator.tsx rename to packages/web/src/app/[domain]/components/navigationMenu/trialIndicator.tsx index 70b13ecd..f7af06fc 100644 --- a/packages/web/src/app/[domain]/components/trialNavIndicator.tsx +++ b/packages/web/src/app/[domain]/components/navigationMenu/trialIndicator.tsx @@ -13,7 +13,7 @@ interface Props { } | null | ServiceError; } -export const TrialNavIndicator = ({ subscription }: Props) => { +export const TrialIndicator = ({ subscription }: Props) => { const domain = useDomain(); const captureEvent = useCaptureEvent(); diff --git a/packages/web/src/app/[domain]/components/pathHeader.tsx b/packages/web/src/app/[domain]/components/pathHeader.tsx index 18806937..11b5bf1d 100644 --- a/packages/web/src/app/[domain]/components/pathHeader.tsx +++ b/packages/web/src/app/[domain]/components/pathHeader.tsx @@ -3,7 +3,7 @@ import { cn, getCodeHostInfoForRepo } from "@/lib/utils"; import { LaptopIcon } from "@radix-ui/react-icons"; import Image from "next/image"; -import { getBrowsePath } from "../browse/hooks/useBrowseNavigation"; +import { getBrowsePath } from "../browse/hooks/utils"; import { ChevronRight, MoreHorizontal } from "lucide-react"; import { useCallback, useState, useMemo, useRef, useEffect } from "react"; import { useToast } from "@/components/hooks/use-toast"; diff --git a/packages/web/src/app/[domain]/components/progressNavIndicator.tsx b/packages/web/src/app/[domain]/components/progressNavIndicator.tsx deleted file mode 100644 index f9e0d8cb..00000000 --- a/packages/web/src/app/[domain]/components/progressNavIndicator.tsx +++ /dev/null @@ -1,73 +0,0 @@ -"use client"; - -import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { useDomain } from "@/hooks/useDomain"; -import { env } from "@/env.mjs"; -import { unwrapServiceError } from "@/lib/utils"; -import { RepoIndexingStatus } from "@prisma/client"; -import { useQuery } from "@tanstack/react-query"; -import { Loader2Icon } from "lucide-react"; -import Link from "next/link"; -import { getRepos } from "@/app/api/(client)/client"; - -export const ProgressNavIndicator = () => { - const domain = useDomain(); - const captureEvent = useCaptureEvent(); - - const { data: inProgressRepos, isPending, isError } = useQuery({ - queryKey: ['repos'], - queryFn: () => unwrapServiceError(getRepos()), - select: (data) => data.filter(repo => repo.repoIndexingStatus === RepoIndexingStatus.IN_INDEX_QUEUE || repo.repoIndexingStatus === RepoIndexingStatus.INDEXING), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - if (isPending || isError || inProgressRepos.length === 0) { - return null; - } - - return ( - captureEvent('wa_progress_nav_pressed', {})} - > - - captureEvent('wa_progress_nav_hover', {})}> -
- - {inProgressRepos.length} -
- - -
-
-
-

Indexing in Progress

-
-

- The following repositories are currently being indexed: -

-
- { - inProgressRepos.slice(0, 10) - .map(item => ( -
- {item.repoName} -
- ) - )} - {inProgressRepos.length > 10 && ( -
- And {inProgressRepos.length - 10} more... -
- )} -
-
- - - - ); -}; \ No newline at end of file diff --git a/packages/web/src/app/[domain]/components/repositoryCarousel.tsx b/packages/web/src/app/[domain]/components/repositoryCarousel.tsx new file mode 100644 index 00000000..bd4f2d1a --- /dev/null +++ b/packages/web/src/app/[domain]/components/repositoryCarousel.tsx @@ -0,0 +1,158 @@ +'use client'; + +import { + Carousel, + CarouselContent, + CarouselItem, +} from "@/components/ui/carousel"; +import { captureEvent } from "@/hooks/useCaptureEvent"; +import { RepositoryQuery } from "@/lib/types"; +import { getCodeHostInfoForRepo } from "@/lib/utils"; +import { FileIcon } from "@radix-ui/react-icons"; +import clsx from "clsx"; +import Autoscroll from "embla-carousel-auto-scroll"; +import Image from "next/image"; +import Link from "next/link"; +import { getBrowsePath } from "../browse/hooks/utils"; +import { useDomain } from "@/hooks/useDomain"; + +interface RepositoryCarouselProps { + displayRepos: RepositoryQuery[]; + numberOfReposWithIndex: number; +} + +export function RepositoryCarousel({ + displayRepos, + numberOfReposWithIndex, +}: RepositoryCarouselProps) { + const domain = useDomain(); + + if (numberOfReposWithIndex === 0) { + return ( +
+ No repositories found + +
+
+ + <> + Create a{" "} + + connection + {" "} + to start indexing repositories + + +
+
+
+ ) + } + + return ( +
+ + {`${numberOfReposWithIndex} `} + + {numberOfReposWithIndex > 1 ? 'repositories' : 'repository'} + + {` indexed`} + + + + {displayRepos.map((repo, index) => ( + + + + ))} + + + {process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT === "demo" && ( +

+ Interested in using Sourcebot on your code? Check out our{' '} + captureEvent('wa_demo_docs_link_pressed', {})} + > + docs + +

+ )} +
+ ) +} + +interface RepositoryBadgeProps { + repo: RepositoryQuery; +} + +const RepositoryBadge = ({ + repo +}: RepositoryBadgeProps) => { + const domain = useDomain(); + const { repoIcon, displayName } = (() => { + const info = getCodeHostInfoForRepo({ + codeHostType: repo.codeHostType, + name: repo.repoName, + displayName: repo.repoDisplayName, + webUrl: repo.webUrl, + }); + + if (info) { + return { + repoIcon: {info.codeHostName}, + displayName: info.displayName, + } + } + + return { + repoIcon: , + displayName: repo.repoName, + } + })(); + + return ( + + {repoIcon} + + {displayName} + + + ) +} diff --git a/packages/web/src/app/[domain]/components/homepage/toolbar.tsx b/packages/web/src/app/[domain]/components/searchModeSelector.tsx similarity index 83% rename from packages/web/src/app/[domain]/components/homepage/toolbar.tsx rename to packages/web/src/app/[domain]/components/searchModeSelector.tsx index cc9c65e0..e615ac8c 100644 --- a/packages/web/src/app/[domain]/components/homepage/toolbar.tsx +++ b/packages/web/src/app/[domain]/components/searchModeSelector.tsx @@ -1,13 +1,16 @@ 'use client'; import { KeyboardShortcutHint } from "@/app/components/keyboardShortcutHint"; +import { useDomain } from "@/hooks/useDomain"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Separator } from "@/components/ui/separator"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { cn } from "@/lib/utils"; -import { MessageCircleIcon, SearchIcon, TriangleAlert } from "lucide-react"; +import { MessageCircleIcon, SearchIcon } from "lucide-react"; import Link from "next/link"; -import { useState } from "react"; +import { useRouter } from "next/navigation"; +import { useCallback, useState } from "react"; +import { useHotkeys } from "react-hotkeys-hook"; export type SearchMode = "precise" | "agentic"; @@ -17,24 +20,47 @@ const AGENTIC_SEARCH_DOCS_URL = "https://docs.sourcebot.dev/docs/features/ask/ov export interface SearchModeSelectorProps { searchMode: SearchMode; - isAgenticSearchEnabled: boolean; - onSearchModeChange: (searchMode: SearchMode) => void; className?: string; } export const SearchModeSelector = ({ searchMode, - isAgenticSearchEnabled, - onSearchModeChange, className, }: SearchModeSelectorProps) => { + const domain = useDomain(); const [focusedSearchMode, setFocusedSearchMode] = useState(searchMode); + const router = useRouter(); + + const onSearchModeChanged = useCallback((value: SearchMode) => { + router.push(`/${domain}/${value === "precise" ? "search" : "chat"}`); + }, [domain, router]); + + useHotkeys("mod+i", (e) => { + e.preventDefault(); + onSearchModeChanged("agentic"); + }, { + enableOnFormTags: true, + enableOnContentEditable: true, + description: "Switch to agentic search", + }); + + useHotkeys("mod+p", (e) => { + e.preventDefault(); + onSearchModeChanged("precise"); + }, { + enableOnFormTags: true, + enableOnContentEditable: true, + description: "Switch to precise search", + }); + return (
- - - - )} - /> -
- -
- - -
- ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/[id]/components/notFoundWarning.tsx b/packages/web/src/app/[domain]/connections/[id]/components/notFoundWarning.tsx deleted file mode 100644 index c65d4f10..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/notFoundWarning.tsx +++ /dev/null @@ -1,80 +0,0 @@ -'use client'; - -import { AlertTriangle } from "lucide-react" -import { Prisma, ConnectionSyncStatus } from "@sourcebot/db" -import { SyncStatusMetadataSchema } from "@/lib/syncStatusMetadataSchema" -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { ReloadIcon } from "@radix-ui/react-icons"; -import { Button } from "@/components/ui/button"; - -interface NotFoundWarningProps { - syncStatus: ConnectionSyncStatus - syncStatusMetadata: Prisma.JsonValue - onSecretsClick: () => void - connectionType: string - onRetrySync: () => void -} - -export const NotFoundWarning = ({ syncStatus, syncStatusMetadata, onSecretsClick, connectionType, onRetrySync }: NotFoundWarningProps) => { - const captureEvent = useCaptureEvent(); - - const parseResult = SyncStatusMetadataSchema.safeParse(syncStatusMetadata); - if (syncStatus !== ConnectionSyncStatus.SYNCED_WITH_WARNINGS || !parseResult.success || !parseResult.data.notFound) { - return null; - } - - const { notFound } = parseResult.data; - - if (notFound.users.length === 0 && notFound.orgs.length === 0 && notFound.repos.length === 0) { - return null; - } else { - captureEvent('wa_connection_not_found_warning_displayed', {}); - } - - return ( -
-
- -

Unable to fetch all references

-
-

- Some requested references couldn't be found. Please ensure you've provided the information listed below correctly, and that you've provided a{" "} - {" "} - to access them if they're private. -

-
    - {notFound.users.length > 0 && ( -
  • - Users: - {notFound.users.join(', ')} -
    • - )} - {notFound.orgs.length > 0 && ( -
  • - {connectionType === "gitlab" ? "Groups" : "Organizations"}: - {notFound.orgs.join(', ')} -
    • - )} - {notFound.repos.length > 0 && ( -
  • - {connectionType === "gitlab" ? "Projects" : "Repositories"}: - {notFound.repos.join(', ')} -
    • - )} -
-
- -
-
- ) -} diff --git a/packages/web/src/app/[domain]/connections/[id]/components/overview.tsx b/packages/web/src/app/[domain]/connections/[id]/components/overview.tsx deleted file mode 100644 index 6b7dbbaf..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/overview.tsx +++ /dev/null @@ -1,159 +0,0 @@ -'use client'; - -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card" -import { DisplayConnectionError } from "./connectionError" -import { NotFoundWarning } from "./notFoundWarning" -import { useDomain } from "@/hooks/useDomain"; -import { useRouter } from "next/navigation"; -import { useCallback } from "react"; -import { useQuery } from "@tanstack/react-query"; -import { flagConnectionForSync, getConnectionInfo } from "@/actions"; -import { isServiceError, unwrapServiceError } from "@/lib/utils"; -import { env } from "@/env.mjs"; -import { ConnectionSyncStatus } from "@sourcebot/db"; -import { FiLoader } from "react-icons/fi"; -import { CircleCheckIcon, AlertTriangle, CircleXIcon } from "lucide-react"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { ReloadIcon } from "@radix-ui/react-icons"; -import { toast } from "@/components/hooks/use-toast"; - -interface OverviewProps { - connectionId: number; -} - -export const Overview = ({ connectionId }: OverviewProps) => { - const captureEvent = useCaptureEvent(); - const domain = useDomain(); - const router = useRouter(); - - const { data: connection, isPending, error, refetch } = useQuery({ - queryKey: ['connection', domain, connectionId], - queryFn: () => unwrapServiceError(getConnectionInfo(connectionId, domain)), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - const handleSecretsNavigation = useCallback(() => { - captureEvent('wa_connection_secrets_navigation_pressed', {}); - router.push(`/${domain}/secrets`); - }, [captureEvent, domain, router]); - - const onRetrySync = useCallback(async () => { - const result = await flagConnectionForSync(connectionId, domain); - if (isServiceError(result)) { - toast({ - description: `❌ Failed to flag connection for sync.`, - }); - captureEvent('wa_connection_retry_sync_fail', { - error: result.errorCode, - }); - } else { - toast({ - description: "✅ Connection flagged for sync.", - }); - captureEvent('wa_connection_retry_sync_success', {}); - refetch(); - } - }, [connectionId, domain, captureEvent, refetch]); - - - if (error) { - return
- {`Error loading connection. Reason: ${error.message}`} -
- } - - if (isPending) { - return ( -
- {Array.from({ length: 4 }).map((_, i) => ( -
-
-
-
- ))} -
- ) - } - - return ( -
-
-
-

Connection Type

-

{connection.connectionType}

-
-
-

Last Synced At

-

- {connection.syncedAt ? new Date(connection.syncedAt).toLocaleDateString() : "never"} -

-
-
-

Linked Repositories

-

{connection.numLinkedRepos}

-
-
-

Status

-
- {connection.syncStatus === "FAILED" ? ( - - captureEvent('wa_connection_failed_status_hover', {})}> - - - - - - - ) : ( - - )} - {connection.syncStatus === "FAILED" && ( - - )} -
-
-
- -
- ) -} - -const SyncStatusBadge = ({ status }: { status: ConnectionSyncStatus }) => { - return ( - - {status === ConnectionSyncStatus.SYNC_NEEDED || status === ConnectionSyncStatus.IN_SYNC_QUEUE ? ( - <> Sync queued - ) : status === ConnectionSyncStatus.SYNCING ? ( - <> Syncing - ) : status === ConnectionSyncStatus.SYNCED ? ( - Synced - ) : status === ConnectionSyncStatus.SYNCED_WITH_WARNINGS ? ( - Synced with warnings - ) : status === ConnectionSyncStatus.FAILED ? ( - <> Sync failed - ) : null} - - ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/[id]/components/repoList.tsx b/packages/web/src/app/[domain]/connections/[id]/components/repoList.tsx deleted file mode 100644 index 962b04cb..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/repoList.tsx +++ /dev/null @@ -1,229 +0,0 @@ -'use client'; - -import { useDomain } from "@/hooks/useDomain"; -import { useQuery } from "@tanstack/react-query"; -import { flagReposForIndex, getConnectionInfo, getRepos } from "@/actions"; -import { RepoListItem } from "./repoListItem"; -import { isServiceError, unwrapServiceError } from "@/lib/utils"; -import { ScrollArea } from "@/components/ui/scroll-area"; -import { ConnectionSyncStatus, RepoIndexingStatus } from "@sourcebot/db"; -import { Search, Loader2 } from "lucide-react"; -import { Input } from "@/components/ui/input"; -import { useCallback, useMemo, useState } from "react"; -import { RepoListItemSkeleton } from "./repoListItemSkeleton"; -import { env } from "@/env.mjs"; -import { Button } from "@/components/ui/button"; -import { useRouter } from "next/navigation"; -import { MultiSelect } from "@/components/ui/multi-select"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { useToast } from "@/components/hooks/use-toast"; - -interface RepoListProps { - connectionId: number; -} - -const getPriority = (status: RepoIndexingStatus) => { - switch (status) { - case RepoIndexingStatus.FAILED: - return 0 - case RepoIndexingStatus.IN_INDEX_QUEUE: - case RepoIndexingStatus.INDEXING: - return 1 - case RepoIndexingStatus.INDEXED: - return 2 - default: - return 3 - } -} - -const convertIndexingStatus = (status: RepoIndexingStatus) => { - switch (status) { - case RepoIndexingStatus.FAILED: - return 'failed'; - case RepoIndexingStatus.NEW: - return 'waiting'; - case RepoIndexingStatus.IN_INDEX_QUEUE: - case RepoIndexingStatus.INDEXING: - return 'running'; - case RepoIndexingStatus.INDEXED: - return 'succeeded'; - default: - return 'unknown'; - } -} - -export const RepoList = ({ connectionId }: RepoListProps) => { - const domain = useDomain(); - const router = useRouter(); - const { toast } = useToast(); - const [searchQuery, setSearchQuery] = useState(""); - const [selectedStatuses, setSelectedStatuses] = useState([]); - const captureEvent = useCaptureEvent(); - const [isRetryAllFailedReposLoading, setIsRetryAllFailedReposLoading] = useState(false); - - const { data: unfilteredRepos, isPending: isReposPending, error: reposError, refetch: refetchRepos } = useQuery({ - queryKey: ['repos', domain, connectionId], - queryFn: async () => { - const repos = await unwrapServiceError(getRepos({ connectionId })); - return repos.sort((a, b) => { - const priorityA = getPriority(a.repoIndexingStatus); - const priorityB = getPriority(b.repoIndexingStatus); - - // First sort by priority - if (priorityA !== priorityB) { - return priorityA - priorityB; - } - - // If same priority, sort by indexedAt - return new Date(a.indexedAt ?? new Date()).getTime() - new Date(b.indexedAt ?? new Date()).getTime(); - }); - }, - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - const { data: connection, isPending: isConnectionPending, error: isConnectionError } = useQuery({ - queryKey: ['connection', domain, connectionId], - queryFn: () => unwrapServiceError(getConnectionInfo(connectionId, domain)), - }) - - - const failedRepos = useMemo(() => { - return unfilteredRepos?.filter((repo) => repo.repoIndexingStatus === RepoIndexingStatus.FAILED) ?? []; - }, [unfilteredRepos]); - - - const onRetryAllFailedRepos = useCallback(() => { - if (failedRepos.length === 0) { - return; - } - - setIsRetryAllFailedReposLoading(true); - flagReposForIndex(failedRepos.map((repo) => repo.repoId)) - .then((response) => { - if (isServiceError(response)) { - captureEvent('wa_connection_retry_all_failed_repos_fail', {}); - toast({ - description: `❌ Failed to flag repositories for indexing. Reason: ${response.message}`, - }); - } else { - captureEvent('wa_connection_retry_all_failed_repos_success', {}); - toast({ - description: `✅ ${failedRepos.length} repositories flagged for indexing.`, - }); - } - }) - .then(() => { refetchRepos() }) - .finally(() => { - setIsRetryAllFailedReposLoading(false); - }); - }, [captureEvent, failedRepos, refetchRepos, toast]); - - const filteredRepos = useMemo(() => { - if (isServiceError(unfilteredRepos)) { - return unfilteredRepos; - } - - const searchLower = searchQuery.toLowerCase(); - return unfilteredRepos?.filter((repo) => { - return repo.repoName.toLowerCase().includes(searchLower); - }).filter((repo) => { - if (selectedStatuses.length === 0) { - return true; - } - - return selectedStatuses.includes(convertIndexingStatus(repo.repoIndexingStatus)); - }); - }, [unfilteredRepos, searchQuery, selectedStatuses]); - - if (reposError) { - return
- {`Error loading repositories. Reason: ${reposError.message}`} -
- } - - return ( -
-
-
- - setSearchQuery(e.target.value)} - /> -
- - setSelectedStatuses(value)} - defaultValue={[]} - placeholder="Filter by status" - maxCount={2} - animation={0} - /> - - {failedRepos.length > 0 && ( - - )} -
- - {isReposPending ? ( -
- {Array.from({ length: 3 }).map((_, i) => ( - - ))} -
- ) : (!filteredRepos || filteredRepos.length === 0) ? ( -
-

No Repositories Found

-

- { - searchQuery.length > 0 ? ( - No repositories found matching your filters. - ) : (!isConnectionError && !isConnectionPending && (connection.syncStatus === ConnectionSyncStatus.IN_SYNC_QUEUE || connection.syncStatus === ConnectionSyncStatus.SYNCING || connection.syncStatus === ConnectionSyncStatus.SYNC_NEEDED)) ? ( - Repositories are being synced. Please check back soon. - ) : ( - - )} -

-
- ) : ( -
- {filteredRepos?.map((repo) => ( - - ))} -
- )} - -
- ) -} diff --git a/packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx b/packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx deleted file mode 100644 index fd491376..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/repoListItem.tsx +++ /dev/null @@ -1,113 +0,0 @@ -'use client'; - -import { getDisplayTime, getRepoImageSrc } from "@/lib/utils"; -import Image from "next/image"; -import { StatusIcon } from "../../components/statusIcon"; -import { RepoIndexingStatus } from "@sourcebot/db"; -import { useMemo } from "react"; -import { RetryRepoIndexButton } from "./repoRetryIndexButton"; - - -interface RepoListItemProps { - name: string; - status: RepoIndexingStatus; - imageUrl?: string; - indexedAt?: Date; - repoId: number; - domain: string; -} - -export const RepoListItem = ({ - imageUrl, - name, - indexedAt, - status, - repoId, - domain, -}: RepoListItemProps) => { - const statusDisplayName = useMemo(() => { - switch (status) { - case RepoIndexingStatus.NEW: - return 'Waiting...'; - case RepoIndexingStatus.IN_INDEX_QUEUE: - return 'In index queue...'; - case RepoIndexingStatus.INDEXING: - return 'Indexing...'; - case RepoIndexingStatus.INDEXED: - return 'Indexed'; - case RepoIndexingStatus.FAILED: - return 'Index failed'; - case RepoIndexingStatus.IN_GC_QUEUE: - return 'In garbage collection queue...'; - case RepoIndexingStatus.GARBAGE_COLLECTING: - return 'Garbage collecting...'; - case RepoIndexingStatus.GARBAGE_COLLECTION_FAILED: - return 'Garbage collection failed'; - } - }, [status]); - - const imageSrc = getRepoImageSrc(imageUrl, repoId, domain); - - return ( -
-
- {imageSrc ? ( - {name} - ) : ( -
- {name.charAt(0)} -
- )} -

{name}

-
-
- {status === RepoIndexingStatus.FAILED && ( - - )} -
- -

- {statusDisplayName} - { - ( - status === RepoIndexingStatus.INDEXED || - status === RepoIndexingStatus.FAILED - ) && indexedAt && ( - {` ${getDisplayTime(indexedAt)}`} - ) - } -

-
-
-
- ) -} - -const convertIndexingStatus = (status: RepoIndexingStatus) => { - switch (status) { - case RepoIndexingStatus.NEW: - return 'waiting'; - case RepoIndexingStatus.IN_INDEX_QUEUE: - case RepoIndexingStatus.INDEXING: - return 'running'; - case RepoIndexingStatus.IN_GC_QUEUE: - case RepoIndexingStatus.GARBAGE_COLLECTING: - return "garbage-collecting" - case RepoIndexingStatus.INDEXED: - return 'succeeded'; - case RepoIndexingStatus.GARBAGE_COLLECTION_FAILED: - case RepoIndexingStatus.FAILED: - return 'failed'; - } -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/[id]/components/repoListItemSkeleton.tsx b/packages/web/src/app/[domain]/connections/[id]/components/repoListItemSkeleton.tsx deleted file mode 100644 index 2eab9a38..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/repoListItemSkeleton.tsx +++ /dev/null @@ -1,15 +0,0 @@ -import { Skeleton } from "@/components/ui/skeleton" - -export const RepoListItemSkeleton = () => { - return ( -
-
- - -
-
- -
-
- ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/[id]/components/repoRetryIndexButton.tsx b/packages/web/src/app/[domain]/connections/[id]/components/repoRetryIndexButton.tsx deleted file mode 100644 index ad44c60e..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/components/repoRetryIndexButton.tsx +++ /dev/null @@ -1,43 +0,0 @@ -"use client"; - -import { Button } from "@/components/ui/button"; -import { ReloadIcon } from "@radix-ui/react-icons" -import { toast } from "@/components/hooks/use-toast"; -import { flagReposForIndex } from "@/actions"; -import { isServiceError } from "@/lib/utils"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; - -interface RetryRepoIndexButtonProps { - repoId: number; -} - -export const RetryRepoIndexButton = ({ repoId }: RetryRepoIndexButtonProps) => { - const captureEvent = useCaptureEvent(); - - return ( - - ); -}; diff --git a/packages/web/src/app/[domain]/connections/[id]/page.tsx b/packages/web/src/app/[domain]/connections/[id]/page.tsx deleted file mode 100644 index 0e0a91d2..00000000 --- a/packages/web/src/app/[domain]/connections/[id]/page.tsx +++ /dev/null @@ -1,63 +0,0 @@ -import { NotFound } from "@/app/[domain]/components/notFound" -import { - Breadcrumb, - BreadcrumbItem, - BreadcrumbLink, - BreadcrumbList, - BreadcrumbPage, - BreadcrumbSeparator, -} from "@/components/ui/breadcrumb" -import { ConnectionIcon } from "../components/connectionIcon" -import { Header } from "../../components/header" -import { RepoList } from "./components/repoList" -import { getConnectionByDomain } from "@/data/connection" -import { Overview } from "./components/overview" - -interface ConnectionManagementPageProps { - params: Promise<{ - domain: string - id: string - }>, -} - -export default async function ConnectionManagementPage(props: ConnectionManagementPageProps) { - const params = await props.params; - const connection = await getConnectionByDomain(Number(params.id), params.domain); - if (!connection) { - return - } - - return ( -
-
- - - - Connections - - - - {connection.name} - - - -
- -

{connection.name}

-
-
- -
-
-

Overview

- -
- -
-

Linked Repositories

- -
-
-
- ) -} diff --git a/packages/web/src/app/[domain]/connections/components/connectionIcon.tsx b/packages/web/src/app/[domain]/connections/components/connectionIcon.tsx deleted file mode 100644 index 2c08121a..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionIcon.tsx +++ /dev/null @@ -1,38 +0,0 @@ -'use client'; - -import { cn, CodeHostType, getCodeHostIcon } from "@/lib/utils"; -import { useMemo } from "react"; -import Image from "next/image"; -import placeholderLogo from "@/public/placeholder_avatar.png"; - -interface ConnectionIconProps { - type: string; - className?: string; -} - -export const ConnectionIcon = ({ - type, - className, -}: ConnectionIconProps) => { - const Icon = useMemo(() => { - const iconInfo = getCodeHostIcon(type as CodeHostType); - if (iconInfo) { - return ( - {`${type} - ) - } - - return {''} - - }, [className, type]); - - return Icon; -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx deleted file mode 100644 index 02c41139..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItem.tsx +++ /dev/null @@ -1,120 +0,0 @@ -import { getDisplayTime } from "@/lib/utils"; -import { useMemo } from "react"; -import { ConnectionIcon } from "../connectionIcon"; -import { ConnectionSyncStatus, Prisma } from "@sourcebot/db"; -import { StatusIcon } from "../statusIcon"; -import { ConnectionListItemErrorIndicator } from "./connectionListItemErrorIndicator"; -import { ConnectionListItemWarningIndicator } from "./connectionListItemWarningIndicator"; -import { ConnectionListItemManageButton } from "./connectionListItemManageButton"; - -const convertSyncStatus = (status: ConnectionSyncStatus) => { - switch (status) { - case ConnectionSyncStatus.SYNC_NEEDED: - return 'waiting'; - case ConnectionSyncStatus.IN_SYNC_QUEUE: - case ConnectionSyncStatus.SYNCING: - return 'running'; - case ConnectionSyncStatus.SYNCED: - return 'succeeded'; - case ConnectionSyncStatus.SYNCED_WITH_WARNINGS: - return 'succeeded-with-warnings'; - case ConnectionSyncStatus.FAILED: - return 'failed'; - } -} - -interface ConnectionListItemProps { - id: string; - name: string; - type: string; - status: ConnectionSyncStatus; - syncStatusMetadata: Prisma.JsonValue; - editedAt: Date; - syncedAt?: Date; - failedRepos?: { repoId: number, repoName: string }[]; - disabled: boolean; -} - -export const ConnectionListItem = ({ - id, - name, - type, - status, - syncStatusMetadata, - editedAt, - syncedAt, - failedRepos, - disabled, -}: ConnectionListItemProps) => { - const statusDisplayName = useMemo(() => { - switch (status) { - case ConnectionSyncStatus.SYNC_NEEDED: - return 'Waiting...'; - case ConnectionSyncStatus.IN_SYNC_QUEUE: - case ConnectionSyncStatus.SYNCING: - return 'Syncing...'; - case ConnectionSyncStatus.SYNCED: - return 'Synced'; - case ConnectionSyncStatus.FAILED: - return 'Sync failed'; - case ConnectionSyncStatus.SYNCED_WITH_WARNINGS: - return null; - } - }, [status]); - - const { notFoundData, displayNotFoundWarning } = useMemo(() => { - if (!syncStatusMetadata || typeof syncStatusMetadata !== 'object' || !('notFound' in syncStatusMetadata)) { - return { notFoundData: null, displayNotFoundWarning: false }; - } - - const notFoundData = syncStatusMetadata.notFound as { - users: string[], - orgs: string[], - repos: string[], - } - - return { notFoundData, displayNotFoundWarning: notFoundData.users.length > 0 || notFoundData.orgs.length > 0 || notFoundData.repos.length > 0 }; - }, [syncStatusMetadata]); - - return ( -
-
- -
-

{name}

- {`Edited ${getDisplayTime(editedAt)}`} -
- - -
-
- -

- {statusDisplayName} - { - ( - status === ConnectionSyncStatus.SYNCED || - status === ConnectionSyncStatus.FAILED - ) && syncedAt && ( - {` ${getDisplayTime(syncedAt)}`} - ) - } -

- -
-
- ) -} diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemErrorIndicator.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemErrorIndicator.tsx deleted file mode 100644 index 26c9ee21..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemErrorIndicator.tsx +++ /dev/null @@ -1,62 +0,0 @@ -'use client' - -import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; -import { CircleX } from "lucide-react"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; - -interface ConnectionListItemErrorIndicatorProps { - failedRepos: { repoId: number; repoName: string; }[] | undefined; - connectionId: string; -} - -export const ConnectionListItemErrorIndicator = ({ - failedRepos, - connectionId -}: ConnectionListItemErrorIndicatorProps) => { - const captureEvent = useCaptureEvent() - - if (!failedRepos || failedRepos.length === 0) return null; - - return ( - - - { - captureEvent('wa_connection_list_item_error_pressed', {}) - window.location.href = `connections/${connectionId}` - }} - onMouseEnter={() => captureEvent('wa_connection_list_item_error_hover', {})} - /> - - -
-
- -

Failed to Index Repositories

-
-
-

- {failedRepos.length} {failedRepos.length === 1 ? 'repository' : 'repositories'} failed to index. This is likely due to temporary server load. -

-
-
- {failedRepos.slice(0, 10).map(repo => ( - {repo.repoName} - ))} - {failedRepos.length > 10 && ( - - And {failedRepos.length - 10} more... - - )} -
-
-

- Navigate to the connection for more details and to retry indexing. -

-
-
- - - ); -}; diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemManageButton.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemManageButton.tsx deleted file mode 100644 index 00b7db27..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemManageButton.tsx +++ /dev/null @@ -1,37 +0,0 @@ -'use client' - -import { Button } from "@/components/ui/button"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { useRouter } from "next/navigation"; -import { useDomain } from "@/hooks/useDomain"; - -interface ConnectionListItemManageButtonProps { - id: string; - disabled: boolean; -} - -export const ConnectionListItemManageButton = ({ - id, - disabled, -}: ConnectionListItemManageButtonProps) => { - const captureEvent = useCaptureEvent() - const router = useRouter(); - const domain = useDomain(); - - return ( - - ); -}; diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemWarningIndicator.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemWarningIndicator.tsx deleted file mode 100644 index 690c9b18..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionList/connectionListItemWarningIndicator.tsx +++ /dev/null @@ -1,78 +0,0 @@ -'use client' - -import { HoverCard, HoverCardContent, HoverCardTrigger } from "@/components/ui/hover-card"; -import { AlertTriangle } from "lucide-react"; -import { NotFoundData } from "@/lib/syncStatusMetadataSchema"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; - - -interface ConnectionListItemWarningIndicatorProps { - notFoundData: NotFoundData | null; - connectionId: string; - type: string; - displayWarning: boolean; -} - -export const ConnectionListItemWarningIndicator = ({ - notFoundData, - connectionId, - type, - displayWarning -}: ConnectionListItemWarningIndicatorProps) => { - const captureEvent = useCaptureEvent() - - if (!notFoundData || !displayWarning) return null; - - return ( - - - { - captureEvent('wa_connection_list_item_warning_pressed', {}) - window.location.href = `connections/${connectionId}` - }} - onMouseEnter={() => captureEvent('wa_connection_list_item_warning_hover', {})} - /> - - -
-
- -

Unable to fetch all references

-
-

- Some requested references couldn't be found. Verify the details below and ensure your connection is using a {" "} - {" "} - that has access to any private references. -

-
    - {notFoundData.users.length > 0 && ( -
  • - Users: - {notFoundData.users.join(', ')} -
    • - )} - {notFoundData.orgs.length > 0 && ( -
  • - {type === "gitlab" ? "Groups" : "Organizations"}: - {notFoundData.orgs.join(', ')} -
    • - )} - {notFoundData.repos.length > 0 && ( -
  • - {type === "gitlab" ? "Projects" : "Repositories"}: - {notFoundData.repos.join(', ')} -
    • - )} -
-
- - - ); -}; \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/components/connectionList/index.tsx b/packages/web/src/app/[domain]/connections/components/connectionList/index.tsx deleted file mode 100644 index 94b4f8be..00000000 --- a/packages/web/src/app/[domain]/connections/components/connectionList/index.tsx +++ /dev/null @@ -1,144 +0,0 @@ -"use client"; -import { useDomain } from "@/hooks/useDomain"; -import { ConnectionListItem } from "./connectionListItem"; -import { cn, unwrapServiceError } from "@/lib/utils"; -import { InfoCircledIcon } from "@radix-ui/react-icons"; -import { getConnections } from "@/actions"; -import { Skeleton } from "@/components/ui/skeleton"; -import { useQuery } from "@tanstack/react-query"; -import { env } from "@/env.mjs"; -import { RepoIndexingStatus, ConnectionSyncStatus } from "@sourcebot/db"; -import { Search } from "lucide-react"; -import { Input } from "@/components/ui/input"; -import { useMemo, useState } from "react"; -import { MultiSelect } from "@/components/ui/multi-select"; - -interface ConnectionListProps { - className?: string; - isDisabled: boolean; -} - -const convertSyncStatus = (status: ConnectionSyncStatus) => { - switch (status) { - case ConnectionSyncStatus.SYNC_NEEDED: - return 'waiting'; - case ConnectionSyncStatus.SYNCING: - return 'running'; - case ConnectionSyncStatus.SYNCED: - return 'succeeded'; - case ConnectionSyncStatus.SYNCED_WITH_WARNINGS: - return 'synced-with-warnings'; - case ConnectionSyncStatus.FAILED: - return 'failed'; - default: - return 'unknown'; - } -} - -export const ConnectionList = ({ - className, - isDisabled, -}: ConnectionListProps) => { - const domain = useDomain(); - const [searchQuery, setSearchQuery] = useState(""); - const [selectedStatuses, setSelectedStatuses] = useState([]); - - const { data: unfilteredConnections, isPending, error } = useQuery({ - queryKey: ['connections', domain], - queryFn: () => unwrapServiceError(getConnections(domain)), - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - }); - - const connections = useMemo(() => { - return unfilteredConnections - ?.filter((connection) => connection.name.toLowerCase().includes(searchQuery.toLowerCase())) - .filter((connection) => { - if (selectedStatuses.length === 0) { - return true; - } - - return selectedStatuses.includes(convertSyncStatus(connection.syncStatus)); - }) - .sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime()) ?? []; - }, [unfilteredConnections, searchQuery, selectedStatuses]); - - if (error) { - return
-

Error loading connections: {error.message}

-
- } - - return ( -
-
-
- - setSearchQuery(e.target.value)} - /> -
- - setSelectedStatuses(value)} - defaultValue={[]} - placeholder="Filter by status" - maxCount={2} - animation={0} - /> - -
- - {isPending ? ( - // Skeleton for loading state -
- {Array.from({ length: 3 }).map((_, i) => ( -
- -
- - -
- -
- ))} -
- ) : connections.length > 0 ? ( - connections - .sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime()) - .map((connection) => ( - repo.repoIndexingStatus === RepoIndexingStatus.FAILED).map((repo) => ({ - repoId: repo.id, - repoName: repo.name, - }))} - disabled={isDisabled} - /> - )) - ) : ( -
- -

No connections

-
- )} -
- ); -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/components/statusIcon.tsx b/packages/web/src/app/[domain]/connections/components/statusIcon.tsx deleted file mode 100644 index 4aad8eb6..00000000 --- a/packages/web/src/app/[domain]/connections/components/statusIcon.tsx +++ /dev/null @@ -1,29 +0,0 @@ -import { cn } from "@/lib/utils"; -import { CircleCheckIcon, CircleXIcon } from "lucide-react"; -import { useMemo } from "react"; -import { FiLoader } from "react-icons/fi"; - -export type Status = 'waiting' | 'running' | 'succeeded' | 'succeeded-with-warnings' | 'garbage-collecting' | 'failed'; - -export const StatusIcon = ({ - status, - className, -}: { status: Status, className?: string }) => { - const Icon = useMemo(() => { - switch (status) { - case 'waiting': - case 'garbage-collecting': - case 'running': - return ; - case 'succeeded': - return ; - case 'failed': - return ; - case 'succeeded-with-warnings': - default: - return null; - } - }, [className, status]); - - return Icon; -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/layout.tsx b/packages/web/src/app/[domain]/connections/layout.tsx deleted file mode 100644 index 9aeb3541..00000000 --- a/packages/web/src/app/[domain]/connections/layout.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import { auth } from "@/auth"; -import { NavigationMenu } from "../components/navigationMenu"; -import { redirect } from "next/navigation"; - -interface LayoutProps { - children: React.ReactNode; - params: Promise<{ domain: string }>; -} - -export default async function Layout( - props: LayoutProps -) { - const params = await props.params; - - const { - domain - } = params; - - const { - children - } = props; - - const session = await auth(); - if (!session) { - return redirect(`/${domain}`); - } - - return ( -
- -
-
{children}
-
-
- ) -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/connections/page.tsx b/packages/web/src/app/[domain]/connections/page.tsx deleted file mode 100644 index fdf3f32c..00000000 --- a/packages/web/src/app/[domain]/connections/page.tsx +++ /dev/null @@ -1,35 +0,0 @@ -import { ConnectionList } from "./components/connectionList"; -import { Header } from "../components/header"; -import { getConnections, getOrgMembership } from "@/actions"; -import { isServiceError } from "@/lib/utils"; -import { notFound, ServiceErrorException } from "@/lib/serviceError"; -import { OrgRole } from "@sourcebot/db"; - -export default async function ConnectionsPage(props: { params: Promise<{ domain: string }> }) { - const params = await props.params; - - const { - domain - } = params; - - const connections = await getConnections(domain); - if (isServiceError(connections)) { - throw new ServiceErrorException(connections); - } - - const membership = await getOrgMembership(domain); - if (isServiceError(membership)) { - throw new ServiceErrorException(notFound()); - } - - return ( -
-
-

Connections

-
- -
- ); -} diff --git a/packages/web/src/app/[domain]/connections/quickActions.tsx b/packages/web/src/app/[domain]/connections/quickActions.tsx deleted file mode 100644 index 5f8008b0..00000000 --- a/packages/web/src/app/[domain]/connections/quickActions.tsx +++ /dev/null @@ -1,575 +0,0 @@ -import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type" -import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; -import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/bitbucket.type"; -import { QuickAction } from "../components/configEditor"; -import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; -import { GerritConnectionConfig } from "@sourcebot/schemas/v3/gerrit.type"; -import { CodeSnippet } from "@/app/components/codeSnippet"; - -export const githubQuickActions: QuickAction[] = [ - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - repos: [ - ...(previous.repos ?? []), - "/" - ] - }), - name: "Add a single repo", - selectionText: "/", - description: ( -
- Add a individual repository to sync with. Ensure the repository is visible to the provided token (if any). - Examples: -
- {[ - "sourcebot/sourcebot", - "vercel/next.js", - "torvalds/linux" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - orgs: [ - ...(previous.orgs ?? []), - "" - ] - }), - name: "Add an organization", - selectionText: "", - description: ( -
- Add an organization to sync with. All repositories in the organization visible to the provided token (if any) will be synced. - Examples: -
- {[ - "commaai", - "sourcebot", - "vercel" - ].map((org) => ( - {org} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - users: [ - ...(previous.users ?? []), - "" - ] - }), - name: "Add a user", - selectionText: "", - description: ( -
- Add a user to sync with. All repositories that the user owns visible to the provided token (if any) will be synced. - Examples: -
- {[ - "jane-doe", - "torvalds", - "octocat" - ].map((org) => ( - {org} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - url: previous.url ?? "https://github.example.com", - }), - name: "Set url to GitHub instance", - selectionText: "https://github.example.com", - description: Set a custom GitHub host. Defaults to https://github.com. - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - repos: [ - ...(previous.exclude?.repos ?? []), - "" - ] - } - }), - name: "Exclude by repo name", - selectionText: "", - description: ( -
- Exclude repositories from syncing by name. Glob patterns are supported. - Examples: -
- {[ - "my-org/docs*", - "my-org/test*" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - topics: [ - ...(previous.exclude?.topics ?? []), - "" - ] - } - }), - name: "Exclude by topic", - selectionText: "", - description: ( -
- Exclude topics from syncing. Only repos that do not match any of the provided topics will be synced. Glob patterns are supported. - Examples: -
- {[ - "docs", - "ci" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - topics: [ - ...(previous.topics ?? []), - "" - ] - }), - name: "Include by topic", - selectionText: "", - description: ( -
- Include repositories by topic. Only repos that match at least one of the provided topics will be synced. Glob patterns are supported. - Examples: -
- {[ - "docs", - "ci" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - archived: true, - } - }), - name: "Exclude archived repos", - description: Exclude archived repositories from syncing. - }, - { - fn: (previous: GithubConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - forks: true, - } - }), - name: "Exclude forked repos", - description: Exclude forked repositories from syncing. - } -]; - -export const gitlabQuickActions: QuickAction[] = [ - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - projects: [ - ...previous.projects ?? [], - "" - ] - }), - name: "Add a project", - selectionText: "", - description: ( -
- Add a individual project to sync with. Ensure the project is visible to the provided token (if any). - Examples: -
- {[ - "gitlab-org/gitlab", - "corp/team-project", - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - users: [ - ...previous.users ?? [], - "" - ] - }), - name: "Add a user", - selectionText: "", - description: ( -
- Add a user to sync with. All projects that the user owns visible to the provided token (if any) will be synced. - Examples: -
- {[ - "jane-doe", - "torvalds" - ].map((org) => ( - {org} - ))} -
-
- ) - }, - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - groups: [ - ...previous.groups ?? [], - "" - ] - }), - name: "Add a group", - selectionText: "", - description: ( -
- Add a group to sync with. All projects in the group (and recursive subgroups) visible to the provided token (if any) will be synced. - Examples: -
- {[ - "my-group", - "path/to/subgroup" - ].map((org) => ( - {org} - ))} -
-
- ) - }, - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - url: previous.url ?? "https://gitlab.example.com", - }), - name: "Set url to GitLab instance", - selectionText: "https://gitlab.example.com", - description: Set a custom GitLab host. Defaults to https://gitlab.com. - }, - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - all: true, - }), - name: "Sync all projects", - description: Sync all projects visible to the provided token (if any). Only available when using a self-hosted GitLab instance. - }, - { - fn: (previous: GitlabConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - projects: [ - ...(previous.exclude?.projects ?? []), - "" - ] - } - }), - name: "Exclude a project", - selectionText: "", - description: ( -
- List of projects to exclude from syncing. Glob patterns are supported. - Examples: -
- {[ - "docs/**", - "**/tests/**", - ].map((repo) => ( - {repo} - ))} -
-
- ) - } -] - -export const giteaQuickActions: QuickAction[] = [ - { - fn: (previous: GiteaConnectionConfig) => ({ - ...previous, - orgs: [ - ...(previous.orgs ?? []), - "" - ] - }), - name: "Add an organization", - selectionText: "", - }, - { - fn: (previous: GiteaConnectionConfig) => ({ - ...previous, - repos: [ - ...(previous.repos ?? []), - "/" - ] - }), - name: "Add a repo", - selectionText: "/", - }, - { - fn: (previous: GiteaConnectionConfig) => ({ - ...previous, - url: previous.url ?? "https://gitea.example.com", - }), - name: "Set url to Gitea instance", - selectionText: "https://gitea.example.com", - } -] - -export const gerritQuickActions: QuickAction[] = [ - { - fn: (previous: GerritConnectionConfig) => ({ - ...previous, - projects: [ - ...(previous.projects ?? []), - "" - ] - }), - name: "Add a project", - }, - { - fn: (previous: GerritConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - projects: [ - ...(previous.exclude?.projects ?? []), - "" - ] - } - }), - name: "Exclude a project", - } -] - -export const bitbucketCloudQuickActions: QuickAction[] = [ - { - // add user - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - user: previous.user ?? "username" - }), - name: "Add username", - selectionText: "username", - description: ( -
- Username to use for authentication. This is only required if you're using an App Password (stored in token) for authentication. -
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - workspaces: [ - ...(previous.workspaces ?? []), - "myWorkspace" - ] - }), - name: "Add a workspace", - selectionText: "myWorkspace", - description: ( -
- Add a workspace to sync with. Ensure the workspace is visible to the provided token (if any). -
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - repos: [ - ...(previous.repos ?? []), - "myWorkspace/myRepo" - ] - }), - name: "Add a repo", - selectionText: "myWorkspace/myRepo", - description: ( -
- Add an individual repository to sync with. Ensure the repository is visible to the provided token (if any). -
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - projects: [ - ...(previous.projects ?? []), - "myProject" - ] - }), - name: "Add a project", - selectionText: "myProject", - description: ( -
- Add a project to sync with. Ensure the project is visible to the provided token (if any). -
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - repos: [...(previous.exclude?.repos ?? []), "myWorkspace/myExcludedRepo"] - } - }), - name: "Exclude a repo", - selectionText: "myWorkspace/myExcludedRepo", - description: ( -
- Exclude a repository from syncing. Glob patterns are supported. -
- ) - }, - // exclude forked - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - forks: true - } - }), - name: "Exclude forked repos", - description: Exclude forked repositories from syncing. - } -] - -export const bitbucketDataCenterQuickActions: QuickAction[] = [ - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - url: previous.url ?? "https://bitbucket.example.com", - }), - name: "Set url to Bitbucket DC instance", - selectionText: "https://bitbucket.example.com", - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - repos: [ - ...(previous.repos ?? []), - "myProject/myRepo" - ] - }), - name: "Add a repo", - selectionText: "myProject/myRepo", - description: ( -
- Add a individual repository to sync with. Ensure the repository is visible to the provided token (if any). - Examples: -
- {[ - "PROJ/repo-name", - "MYPROJ/api" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - projects: [ - ...(previous.projects ?? []), - "myProject" - ] - }), - name: "Add a project", - selectionText: "myProject", - description: ( -
- Add a project to sync with. Ensure the project is visible to the provided token (if any). -
- ) - }, - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - repos: [...(previous.exclude?.repos ?? []), "myProject/myExcludedRepo"] - } - }), - name: "Exclude a repo", - selectionText: "myProject/myExcludedRepo", - description: ( -
- Exclude a repository from syncing. Glob patterns are supported. - Examples: -
- {[ - "myProject/myExcludedRepo", - "myProject2/*" - ].map((repo) => ( - {repo} - ))} -
-
- ) - }, - // exclude archived - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - archived: true - } - }), - name: "Exclude archived repos", - }, - // exclude forked - { - fn: (previous: BitbucketConnectionConfig) => ({ - ...previous, - exclude: { - ...previous.exclude, - forks: true - } - }), - name: "Exclude forked repos", - } -] - diff --git a/packages/web/src/app/[domain]/connections/utils.ts b/packages/web/src/app/[domain]/connections/utils.ts deleted file mode 100644 index cd728092..00000000 --- a/packages/web/src/app/[domain]/connections/utils.ts +++ /dev/null @@ -1,40 +0,0 @@ -import Ajv, { Schema } from "ajv"; -import { z } from "zod"; - -export const createZodConnectionConfigValidator = (jsonSchema: Schema, additionalConfigValidation?: (config: T) => { message: string, isValid: boolean }) => { - const ajv = new Ajv({ - validateFormats: false, - }); - const validate = ajv.compile(jsonSchema); - - return z - .string() - .superRefine((data, ctx) => { - const addIssue = (message: string) => { - return ctx.addIssue({ - code: "custom", - message: `Schema validation error: ${message}` - }); - } - - let parsed; - try { - parsed = JSON.parse(data); - } catch { - addIssue("Invalid JSON"); - return; - } - - const valid = validate(parsed); - if (!valid) { - addIssue(ajv.errorsText(validate.errors)); - } - - if (additionalConfigValidation) { - const result = additionalConfigValidation(parsed as T); - if (!result.isValid) { - addIssue(result.message); - } - } - }); -} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/layout.tsx b/packages/web/src/app/[domain]/layout.tsx index 06dca7d8..b60a50a6 100644 --- a/packages/web/src/app/[domain]/layout.tsx +++ b/packages/web/src/app/[domain]/layout.tsx @@ -22,6 +22,7 @@ import { getAnonymousAccessStatus, getMemberApprovalRequired } from "@/actions"; import { JoinOrganizationCard } from "@/app/components/joinOrganizationCard"; import { LogoutEscapeHatch } from "@/app/components/logoutEscapeHatch"; import { GitHubStarToast } from "./components/githubStarToast"; +import { UpgradeToast } from "./components/upgradeToast"; interface LayoutProps { children: React.ReactNode, @@ -154,6 +155,7 @@ export default async function Layout(props: LayoutProps) { {children} + ) } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/page.tsx b/packages/web/src/app/[domain]/page.tsx index 7fd66f43..1ff9b8d6 100644 --- a/packages/web/src/app/[domain]/page.tsx +++ b/packages/web/src/app/[domain]/page.tsx @@ -1,101 +1,11 @@ -import { getRepos, getSearchContexts } from "@/actions"; -import { Footer } from "@/app/components/footer"; -import { getOrgFromDomain } from "@/data/org"; -import { getConfiguredLanguageModelsInfo, getUserChatHistory } from "@/features/chat/actions"; -import { isServiceError, measure } from "@/lib/utils"; -import { Homepage } from "./components/homepage"; -import { NavigationMenu } from "./components/navigationMenu"; -import { PageNotFound } from "./components/pageNotFound"; -import { UpgradeToast } from "./components/upgradeToast"; -import { ServiceErrorException } from "@/lib/serviceError"; -import { auth } from "@/auth"; -import { cookies } from "next/headers"; -import { AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME, SEARCH_MODE_COOKIE_NAME } from "@/lib/constants"; -import { env } from "@/env.mjs"; -import { loadJsonFile } from "@sourcebot/shared"; -import { DemoExamples, demoExamplesSchema } from "@/types"; -import { createLogger } from "@sourcebot/logger"; +import SearchPage from "./search/page"; -const logger = createLogger('web-homepage'); - -export default async function Home(props: { params: Promise<{ domain: string }> }) { - logger.debug('Starting homepage load...'); - const { data: HomePage, durationMs } = await measure(() => HomeInternal(props), 'HomeInternal', /* outputLog = */ false); - logger.debug(`Homepage load completed in ${durationMs}ms.`); - - return HomePage; +interface Props { + params: Promise<{ domain: string }>; + searchParams: Promise<{ query?: string }>; } -const HomeInternal = async (props: { params: Promise<{ domain: string }> }) => { - const params = await props.params; - - const { - domain - } = params; - - - const org = (await measure(() => getOrgFromDomain(domain), 'getOrgFromDomain')).data; - if (!org) { - return - } - - const session = (await measure(() => auth(), 'auth')).data; - const models = (await measure(() => getConfiguredLanguageModelsInfo(), 'getConfiguredLanguageModelsInfo')).data; - const repos = (await measure(() => getRepos(), 'getRepos')).data; - const searchContexts = (await measure(() => getSearchContexts(domain), 'getSearchContexts')).data; - const chatHistory = session ? (await measure(() => getUserChatHistory(domain), 'getUserChatHistory')).data : []; - - if (isServiceError(repos)) { - throw new ServiceErrorException(repos); - } - - if (isServiceError(searchContexts)) { - throw new ServiceErrorException(searchContexts); - } - - if (isServiceError(chatHistory)) { - throw new ServiceErrorException(chatHistory); - } - - const indexedRepos = repos.filter((repo) => repo.indexedAt !== undefined); - - // Read search mode from cookie, defaulting to agentic if not set - // (assuming a language model is configured). - const cookieStore = (await measure(() => cookies(), 'cookies')).data; - const searchModeCookie = cookieStore.get(SEARCH_MODE_COOKIE_NAME); - const initialSearchMode = ( - searchModeCookie?.value === "agentic" || - searchModeCookie?.value === "precise" - ) ? searchModeCookie.value : models.length > 0 ? "agentic" : "precise"; - - const isAgenticSearchTutorialDismissed = cookieStore.get(AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME)?.value === "true"; - - const demoExamples = env.SOURCEBOT_DEMO_EXAMPLES_PATH ? await (async () => { - try { - return (await measure(() => loadJsonFile(env.SOURCEBOT_DEMO_EXAMPLES_PATH!, demoExamplesSchema), 'loadExamplesJsonFile')).data; - } catch (error) { - console.error('Failed to load demo examples:', error); - return undefined; - } - })() : undefined; - - return ( -
- - - - -
-
- ) +export default async function Home(props: Props) { + // Default to rendering the search page. + return ; } \ No newline at end of file diff --git a/packages/web/src/app/[domain]/repos/columns.tsx b/packages/web/src/app/[domain]/repos/columns.tsx index ca37a9b4..7d617cda 100644 --- a/packages/web/src/app/[domain]/repos/columns.tsx +++ b/packages/web/src/app/[domain]/repos/columns.tsx @@ -2,72 +2,51 @@ import { Button } from "@/components/ui/button" import type { ColumnDef } from "@tanstack/react-table" -import { ArrowUpDown, Clock, Loader2, CheckCircle2, XCircle, Trash2, Check, ListFilter } from "lucide-react" +import { ArrowUpDown, Clock, Loader2, CheckCircle2, Check, ListFilter } from "lucide-react" import Image from "next/image" import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" import { cn, getRepoImageSrc } from "@/lib/utils" -import { RepoIndexingStatus } from "@sourcebot/db"; import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu" import Link from "next/link" -import { getBrowsePath } from "../browse/hooks/useBrowseNavigation" +import { getBrowsePath } from "../browse/hooks/utils" + +export type RepoStatus = 'syncing' | 'indexed' | 'not-indexed'; export type RepositoryColumnInfo = { repoId: number repoName: string; repoDisplayName: string imageUrl?: string - repoIndexingStatus: RepoIndexingStatus + status: RepoStatus lastIndexed: string } -const statusLabels = { - [RepoIndexingStatus.NEW]: "Queued", - [RepoIndexingStatus.IN_INDEX_QUEUE]: "Queued", - [RepoIndexingStatus.INDEXING]: "Indexing", - [RepoIndexingStatus.INDEXED]: "Indexed", - [RepoIndexingStatus.FAILED]: "Failed", - [RepoIndexingStatus.IN_GC_QUEUE]: "Deleting", - [RepoIndexingStatus.GARBAGE_COLLECTING]: "Deleting", - [RepoIndexingStatus.GARBAGE_COLLECTION_FAILED]: "Deletion Failed" +const statusLabels: Record = { + 'syncing': "Syncing", + 'indexed': "Indexed", + 'not-indexed': "Pending", }; -const StatusIndicator = ({ status }: { status: RepoIndexingStatus }) => { +const StatusIndicator = ({ status }: { status: RepoStatus }) => { let icon = null let description = "" let className = "" switch (status) { - case RepoIndexingStatus.NEW: - case RepoIndexingStatus.IN_INDEX_QUEUE: - icon = - description = "Repository is queued for indexing" - className = "text-yellow-600 bg-yellow-50 dark:bg-yellow-900/20 dark:text-yellow-400" - break - case RepoIndexingStatus.INDEXING: + case 'syncing': icon = - description = "Repository is being indexed" + description = "Repository is currently syncing" className = "text-blue-600 bg-blue-50 dark:bg-blue-900/20 dark:text-blue-400" break - case RepoIndexingStatus.INDEXED: + case 'indexed': icon = - description = "Repository has been successfully indexed" + description = "Repository has been successfully indexed and is up to date" className = "text-green-600 bg-green-50 dark:bg-green-900/20 dark:text-green-400" break - case RepoIndexingStatus.FAILED: - icon = - description = "Repository indexing failed" - className = "text-red-600 bg-red-50 dark:bg-red-900/20 dark:text-red-400" - break - case RepoIndexingStatus.IN_GC_QUEUE: - case RepoIndexingStatus.GARBAGE_COLLECTING: - icon = - description = "Repository is being deleted" - className = "text-gray-600 bg-gray-50 dark:bg-gray-900/20 dark:text-gray-400" - break - case RepoIndexingStatus.GARBAGE_COLLECTION_FAILED: - icon = - description = "Repository deletion failed" - className = "text-red-600 bg-red-50 dark:bg-red-900/20 dark:text-red-400" + case 'not-indexed': + icon = + description = "Repository is pending initial sync" + className = "text-yellow-600 bg-yellow-50 dark:bg-yellow-900/20 dark:text-yellow-400" break } @@ -94,6 +73,7 @@ export const columns = (domain: string): ColumnDef[] => [ { accessorKey: "repoDisplayName", header: 'Repository', + size: 500, cell: ({ row: { original: { repoId, repoName, repoDisplayName, imageUrl } } }) => { return (
@@ -130,9 +110,10 @@ export const columns = (domain: string): ColumnDef[] => [ }, }, { - accessorKey: "repoIndexingStatus", + accessorKey: "status", + size: 150, header: ({ column }) => { - const uniqueLabels = Array.from(new Set(Object.values(statusLabels))); + const uniqueLabels = Object.values(statusLabels); const currentFilter = column.getFilterValue() as string | undefined; return ( @@ -173,17 +154,18 @@ export const columns = (domain: string): ColumnDef[] => [ ) }, cell: ({ row }) => { - return + return }, filterFn: (row, id, value) => { if (value === undefined) return true; - const status = row.getValue(id) as RepoIndexingStatus; + const status = row.getValue(id) as RepoStatus; return statusLabels[status] === value; }, }, { accessorKey: "lastIndexed", + size: 150, header: ({ column }) => (
), cell: ({ row }) => { if (!row.original.lastIndexed) { - return
-
; + return
Never
; } const date = new Date(row.original.lastIndexed) return ( diff --git a/packages/web/src/app/[domain]/repos/layout.tsx b/packages/web/src/app/[domain]/repos/layout.tsx index 85d607eb..6ff10adf 100644 --- a/packages/web/src/app/[domain]/repos/layout.tsx +++ b/packages/web/src/app/[domain]/repos/layout.tsx @@ -9,14 +9,8 @@ export default async function Layout( props: LayoutProps ) { const params = await props.params; - - const { - domain - } = params; - - const { - children - } = props; + const { domain } = params; + const { children } = props; return (
diff --git a/packages/web/src/app/[domain]/repos/page.tsx b/packages/web/src/app/[domain]/repos/page.tsx index 4502dafc..e9dd4e43 100644 --- a/packages/web/src/app/[domain]/repos/page.tsx +++ b/packages/web/src/app/[domain]/repos/page.tsx @@ -1,8 +1,22 @@ -import { RepositoryTable } from "./repositoryTable"; -import { getOrgFromDomain } from "@/data/org"; -import { PageNotFound } from "../components/pageNotFound"; -import { Header } from "../components/header"; import { env } from "@/env.mjs"; +import { RepoIndexingJob } from "@sourcebot/db"; +import { Header } from "../components/header"; +import { RepoStatus } from "./columns"; +import { RepositoryTable } from "./repositoryTable"; +import { sew } from "@/actions"; +import { withOptionalAuthV2 } from "@/withAuthV2"; +import { isServiceError } from "@/lib/utils"; +import { ServiceErrorException } from "@/lib/serviceError"; + +function getRepoStatus(repo: { indexedAt: Date | null, jobs: RepoIndexingJob[] }): RepoStatus { + const latestJob = repo.jobs[0]; + + if (latestJob?.status === 'PENDING' || latestJob?.status === 'IN_PROGRESS') { + return 'syncing'; + } + + return repo.indexedAt ? 'indexed' : 'not-indexed'; +} export default async function ReposPage(props: { params: Promise<{ domain: string }> }) { const params = await props.params; @@ -11,9 +25,9 @@ export default async function ReposPage(props: { params: Promise<{ domain: strin domain } = params; - const org = await getOrgFromDomain(domain); - if (!org) { - return + const repos = await getReposWithJobs(); + if (isServiceError(repos)) { + throw new ServiceErrorException(repos); } return ( @@ -21,13 +35,31 @@ export default async function ReposPage(props: { params: Promise<{ domain: strin

Repositories

-
-
- -
+
+ ({ + repoId: repo.id, + repoName: repo.name, + repoDisplayName: repo.displayName ?? repo.name, + imageUrl: repo.imageUrl ?? undefined, + indexedAt: repo.indexedAt ?? undefined, + status: getRepoStatus(repo), + }))} + domain={domain} + isAddReposButtonVisible={env.EXPERIMENT_SELF_SERVE_REPO_INDEXING_ENABLED === 'true'} + />
) } + +const getReposWithJobs = async () => sew(() => + withOptionalAuthV2(async ({ prisma }) => { + const repos = await prisma.repo.findMany({ + include: { + jobs: true, + } + }); + + return repos; + })); \ No newline at end of file diff --git a/packages/web/src/app/[domain]/repos/repositoryTable.tsx b/packages/web/src/app/[domain]/repos/repositoryTable.tsx index 8d9dc0f1..9b67cca7 100644 --- a/packages/web/src/app/[domain]/repos/repositoryTable.tsx +++ b/packages/web/src/app/[domain]/repos/repositoryTable.tsx @@ -1,118 +1,81 @@ "use client"; -import { DataTable } from "@/components/ui/data-table"; -import { columns, RepositoryColumnInfo } from "./columns"; -import { unwrapServiceError } from "@/lib/utils"; -import { useQuery } from "@tanstack/react-query"; -import { useDomain } from "@/hooks/useDomain"; -import { RepoIndexingStatus } from "@sourcebot/db"; -import { useMemo } from "react"; -import { Skeleton } from "@/components/ui/skeleton"; -import { env } from "@/env.mjs"; +import { useToast } from "@/components/hooks/use-toast"; import { Button } from "@/components/ui/button"; -import { PlusIcon } from "lucide-react"; +import { DataTable } from "@/components/ui/data-table"; +import { PlusIcon, RefreshCwIcon } from "lucide-react"; +import { useRouter } from "next/navigation"; +import { useMemo, useState } from "react"; +import { columns, RepositoryColumnInfo, RepoStatus } from "./columns"; import { AddRepositoryDialog } from "./components/addRepositoryDialog"; -import { useState } from "react"; -import { getRepos } from "@/app/api/(client)/client"; interface RepositoryTableProps { - isAddReposButtonVisible: boolean + repos: { + repoId: number; + repoName: string; + repoDisplayName: string; + imageUrl?: string; + indexedAt?: Date; + status: RepoStatus; + }[]; + domain: string; + isAddReposButtonVisible: boolean; } export const RepositoryTable = ({ + repos, + domain, isAddReposButtonVisible, }: RepositoryTableProps) => { - const domain = useDomain(); const [isAddDialogOpen, setIsAddDialogOpen] = useState(false); - - const { data: repos, isLoading: reposLoading, error: reposError } = useQuery({ - queryKey: ['repos'], - queryFn: async () => { - return await unwrapServiceError(getRepos()); - }, - refetchInterval: env.NEXT_PUBLIC_POLLING_INTERVAL_MS, - refetchIntervalInBackground: true, - }); + const router = useRouter(); + const { toast } = useToast(); const tableRepos = useMemo(() => { - if (reposLoading) return Array(4).fill(null).map(() => ({ - repoId: 0, - repoName: "", - repoDisplayName: "", - repoIndexingStatus: RepoIndexingStatus.NEW, - lastIndexed: "", - imageUrl: "", - })); - - if (!repos) return []; return repos.map((repo): RepositoryColumnInfo => ({ repoId: repo.repoId, repoName: repo.repoName, repoDisplayName: repo.repoDisplayName ?? repo.repoName, imageUrl: repo.imageUrl, - repoIndexingStatus: repo.repoIndexingStatus as RepoIndexingStatus, + status: repo.status, lastIndexed: repo.indexedAt?.toISOString() ?? "", })).sort((a, b) => { - const getPriorityFromStatus = (status: RepoIndexingStatus) => { + const getPriorityFromStatus = (status: RepoStatus) => { switch (status) { - case RepoIndexingStatus.IN_INDEX_QUEUE: - case RepoIndexingStatus.INDEXING: - return 0 // Highest priority - currently indexing - case RepoIndexingStatus.FAILED: - return 1 // Second priority - failed repos need attention - case RepoIndexingStatus.INDEXED: + case 'syncing': + return 0 // Highest priority - currently syncing + case 'not-indexed': + return 1 // Second priority - not yet indexed + case 'indexed': return 2 // Third priority - successfully indexed default: - return 3 // Lowest priority - other statuses (NEW, etc.) + return 3 } } // Sort by priority first - const aPriority = getPriorityFromStatus(a.repoIndexingStatus); - const bPriority = getPriorityFromStatus(b.repoIndexingStatus); - + const aPriority = getPriorityFromStatus(a.status); + const bPriority = getPriorityFromStatus(b.status); + if (aPriority !== bPriority) { - return aPriority - bPriority; // Lower priority number = higher precedence + return aPriority - bPriority; } - + // If same priority, sort by last indexed date (most recent first) - return new Date(b.lastIndexed).getTime() - new Date(a.lastIndexed).getTime(); + if (a.lastIndexed && b.lastIndexed) { + return new Date(b.lastIndexed).getTime() - new Date(a.lastIndexed).getTime(); + } + + // Put items without dates at the end + if (!a.lastIndexed) return 1; + if (!b.lastIndexed) return -1; + return 0; }); - }, [repos, reposLoading]); + }, [repos]); const tableColumns = useMemo(() => { - if (reposLoading) { - return columns(domain).map((column) => { - if ('accessorKey' in column && column.accessorKey === "name") { - return { - ...column, - cell: () => ( -
- {/* Avatar skeleton */} - {/* Repository name skeleton */} -
- ), - } - } - - return { - ...column, - cell: () => ( -
- -
- ), - } - }) - } - return columns(domain); - }, [reposLoading, domain]); - - - if (reposError) { - return
Error loading repositories
; - } + }, [domain]); return ( <> @@ -121,18 +84,35 @@ export const RepositoryTable = ({ data={tableRepos} searchKey="repoDisplayName" searchPlaceholder="Search repositories..." - headerActions={isAddReposButtonVisible && ( - + headerActions={( +
+ + {isAddReposButtonVisible && ( + + )} +
)} /> - + { + const carouselRepos = await getRepos({ + where: { + indexedAt: { + not: null, + }, + }, + take: 10, + }); + + const repoStats = await getReposStats(); + + if (isServiceError(carouselRepos)) throw new ServiceErrorException(carouselRepos); + if (isServiceError(repoStats)) throw new ServiceErrorException(repoStats); + + return ( +
+ + +
+
+ +
+
+ + +
+ +
+
+ +
+ +
+ +
+ + How to search +
+ + + test todo (both test and todo) + + + test or todo (either test or todo) + + + {`"exit boot"`} (exact match) + + + TODO case:yes (case sensitive) + + + + + file:README setup (by filename) + + + repo:torvalds/linux test (by repo) + + + lang:typescript (by language) + + + rev:HEAD (by branch or tag) + + + + + file:{`\\.py$`} {`(files that end in ".py")`} + + + sym:main {`(symbols named "main")`} + + + todo -lang:c (negate filter) + + + content:README (search content only) + + +
+ +
+
+
+ ) +} + +const HowToSection = ({ title, children }: { title: string, children: React.ReactNode }) => { + return ( +
+ {title} + {children} +
+ ) + +} + +const Highlight = ({ children }: { children: React.ReactNode }) => { + return ( + + {children} + + ) +} + +const QueryExample = ({ children }: { children: React.ReactNode }) => { + return ( + + {children} + + ) +} + +const QueryExplanation = ({ children }: { children: React.ReactNode }) => { + return ( + + {children} + + ) +} + +const Query = ({ query, domain, children }: { query: string, domain: string, children: React.ReactNode }) => { + return ( + + {children} + + ) +} \ No newline at end of file diff --git a/packages/web/src/app/[domain]/search/components/searchResultsPage.tsx b/packages/web/src/app/[domain]/search/components/searchResultsPage.tsx new file mode 100644 index 00000000..553ee132 --- /dev/null +++ b/packages/web/src/app/[domain]/search/components/searchResultsPage.tsx @@ -0,0 +1,372 @@ +'use client'; + +import { CodeSnippet } from "@/app/components/codeSnippet"; +import { KeyboardShortcutHint } from "@/app/components/keyboardShortcutHint"; +import { useToast } from "@/components/hooks/use-toast"; +import { AnimatedResizableHandle } from "@/components/ui/animatedResizableHandle"; +import { Button } from "@/components/ui/button"; +import { + ResizablePanel, + ResizablePanelGroup, +} from "@/components/ui/resizable"; +import { Separator } from "@/components/ui/separator"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { RepositoryInfo, SearchResultFile, SearchStats } from "@/features/search/types"; +import useCaptureEvent from "@/hooks/useCaptureEvent"; +import { useDomain } from "@/hooks/useDomain"; +import { useNonEmptyQueryParam } from "@/hooks/useNonEmptyQueryParam"; +import { useSearchHistory } from "@/hooks/useSearchHistory"; +import { SearchQueryParams } from "@/lib/types"; +import { createPathWithQueryParams, measure, unwrapServiceError } from "@/lib/utils"; +import { InfoCircledIcon, SymbolIcon } from "@radix-ui/react-icons"; +import { useQuery } from "@tanstack/react-query"; +import { useLocalStorage } from "@uidotdev/usehooks"; +import { AlertTriangleIcon, BugIcon, FilterIcon } from "lucide-react"; +import { useRouter } from "next/navigation"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useHotkeys } from "react-hotkeys-hook"; +import { ImperativePanelHandle } from "react-resizable-panels"; +import { search } from "../../../api/(client)/client"; +import { CopyIconButton } from "../../components/copyIconButton"; +import { SearchBar } from "../../components/searchBar"; +import { TopBar } from "../../components/topBar"; +import { CodePreviewPanel } from "./codePreviewPanel"; +import { FilterPanel } from "./filterPanel"; +import { useFilteredMatches } from "./filterPanel/useFilterMatches"; +import { SearchResultsPanel } from "./searchResultsPanel"; + +const DEFAULT_MAX_MATCH_COUNT = 5000; + +interface SearchResultsPageProps { + searchQuery: string; +} + +export const SearchResultsPage = ({ + searchQuery, +}: SearchResultsPageProps) => { + const router = useRouter(); + const { setSearchHistory } = useSearchHistory(); + const captureEvent = useCaptureEvent(); + const domain = useDomain(); + const { toast } = useToast(); + + // Encodes the number of matches to return in the search response. + const _maxMatchCount = parseInt(useNonEmptyQueryParam(SearchQueryParams.matches) ?? `${DEFAULT_MAX_MATCH_COUNT}`); + const maxMatchCount = isNaN(_maxMatchCount) ? DEFAULT_MAX_MATCH_COUNT : _maxMatchCount; + + const { + data: searchResponse, + isPending: isSearchPending, + isFetching: isFetching, + error + } = useQuery({ + queryKey: ["search", searchQuery, maxMatchCount], + queryFn: () => measure(() => unwrapServiceError(search({ + query: searchQuery, + matches: maxMatchCount, + contextLines: 3, + whole: false, + }, domain)), "client.search"), + select: ({ data, durationMs }) => ({ + ...data, + totalClientSearchDurationMs: durationMs, + }), + enabled: searchQuery.length > 0, + refetchOnWindowFocus: false, + retry: false, + staleTime: 0, + }); + + useEffect(() => { + if (error) { + toast({ + description: `❌ Search failed. Reason: ${error.message}`, + }); + } + }, [error, toast]); + + + // Write the query to the search history + useEffect(() => { + if (searchQuery.length === 0) { + return; + } + + const now = new Date().toUTCString(); + setSearchHistory((searchHistory) => [ + { + query: searchQuery, + date: now, + }, + ...searchHistory.filter(search => search.query !== searchQuery), + ]) + }, [searchQuery, setSearchHistory]); + + useEffect(() => { + if (!searchResponse) { + return; + } + + const fileLanguages = searchResponse.files?.map(file => file.language) || []; + + captureEvent("search_finished", { + durationMs: searchResponse.totalClientSearchDurationMs, + fileCount: searchResponse.stats.fileCount, + matchCount: searchResponse.stats.totalMatchCount, + actualMatchCount: searchResponse.stats.actualMatchCount, + filesSkipped: searchResponse.stats.filesSkipped, + contentBytesLoaded: searchResponse.stats.contentBytesLoaded, + indexBytesLoaded: searchResponse.stats.indexBytesLoaded, + crashes: searchResponse.stats.crashes, + shardFilesConsidered: searchResponse.stats.shardFilesConsidered, + filesConsidered: searchResponse.stats.filesConsidered, + filesLoaded: searchResponse.stats.filesLoaded, + shardsScanned: searchResponse.stats.shardsScanned, + shardsSkipped: searchResponse.stats.shardsSkipped, + shardsSkippedFilter: searchResponse.stats.shardsSkippedFilter, + ngramMatches: searchResponse.stats.ngramMatches, + ngramLookups: searchResponse.stats.ngramLookups, + wait: searchResponse.stats.wait, + matchTreeConstruction: searchResponse.stats.matchTreeConstruction, + matchTreeSearch: searchResponse.stats.matchTreeSearch, + regexpsConsidered: searchResponse.stats.regexpsConsidered, + flushReason: searchResponse.stats.flushReason, + fileLanguages, + }); + }, [captureEvent, searchQuery, searchResponse]); + + + const onLoadMoreResults = useCallback(() => { + const url = createPathWithQueryParams(`/${domain}/search`, + [SearchQueryParams.query, searchQuery], + [SearchQueryParams.matches, `${maxMatchCount * 2}`], + ) + router.push(url); + }, [maxMatchCount, router, searchQuery, domain]); + + return ( +
+ {/* TopBar */} + + + + + {(isSearchPending || isFetching) ? ( +
+ +

Searching...

+
+ ) : error ? ( +
+ +

Failed to search

+

{error.message}

+
+ ) : ( + + )} +
+ ); +} + +interface PanelGroupProps { + fileMatches: SearchResultFile[]; + isMoreResultsButtonVisible?: boolean; + onLoadMoreResults: () => void; + isBranchFilteringEnabled: boolean; + repoInfo: RepositoryInfo[]; + searchDurationMs: number; + numMatches: number; + searchStats?: SearchStats; +} + +const PanelGroup = ({ + fileMatches, + isMoreResultsButtonVisible, + onLoadMoreResults, + isBranchFilteringEnabled, + repoInfo: _repoInfo, + searchDurationMs: _searchDurationMs, + numMatches, + searchStats, +}: PanelGroupProps) => { + const [previewedFile, setPreviewedFile] = useState(undefined); + const filteredFileMatches = useFilteredMatches(fileMatches); + const filterPanelRef = useRef(null); + const [selectedMatchIndex, setSelectedMatchIndex] = useState(0); + + const [isFilterPanelCollapsed, setIsFilterPanelCollapsed] = useLocalStorage('isFilterPanelCollapsed', false); + + useHotkeys("mod+b", () => { + if (isFilterPanelCollapsed) { + filterPanelRef.current?.expand(); + } else { + filterPanelRef.current?.collapse(); + } + }, { + enableOnFormTags: true, + enableOnContentEditable: true, + description: "Toggle filter panel", + }); + + const searchDurationMs = useMemo(() => { + return Math.round(_searchDurationMs); + }, [_searchDurationMs]); + + const repoInfo = useMemo(() => { + return _repoInfo.reduce((acc, repo) => { + acc[repo.id] = repo; + return acc; + }, {} as Record); + }, [_repoInfo]); + + return ( + + {/* ~~ Filter panel ~~ */} + setIsFilterPanelCollapsed(true)} + onExpand={() => setIsFilterPanelCollapsed(false)} + > + + + {isFilterPanelCollapsed && ( +
+ + + + + + + + Open filter panel + + +
+ )} + + + {/* ~~ Search results ~~ */} + +
+ + + + + +
+ +

Search stats for nerds

+ { + navigator.clipboard.writeText(JSON.stringify(searchStats, null, 2)); + return true; + }} + className="ml-auto" + /> +
+ + {JSON.stringify(searchStats, null, 2)} + + + + { + fileMatches.length > 0 ? ( +

{`[${searchDurationMs} ms] Found ${numMatches} matches in ${fileMatches.length} ${fileMatches.length > 1 ? 'files' : 'file'}`}

+ ) : ( +

No results

+ ) + } + {isMoreResultsButtonVisible && ( +
+ (load more) +
+ )} +
+ {filteredFileMatches.length > 0 ? ( + { + setSelectedMatchIndex(matchIndex ?? 0); + setPreviewedFile(fileMatch); + }} + isLoadMoreButtonVisible={!!isMoreResultsButtonVisible} + onLoadMoreButtonClicked={onLoadMoreResults} + isBranchFilteringEnabled={isBranchFilteringEnabled} + repoInfo={repoInfo} + /> + ) : ( +
+

No results found

+
+ )} + + + {previewedFile && ( + <> + + {/* ~~ Code preview ~~ */} + setPreviewedFile(undefined)} + > + setPreviewedFile(undefined)} + selectedMatchIndex={selectedMatchIndex} + onSelectedMatchIndexChange={setSelectedMatchIndex} + /> + + + )} + + ) +} diff --git a/packages/web/src/app/[domain]/search/components/searchResultsPanel/fileMatch.tsx b/packages/web/src/app/[domain]/search/components/searchResultsPanel/fileMatch.tsx index 3b1943ba..d6e6b8ab 100644 --- a/packages/web/src/app/[domain]/search/components/searchResultsPanel/fileMatch.tsx +++ b/packages/web/src/app/[domain]/search/components/searchResultsPanel/fileMatch.tsx @@ -3,7 +3,7 @@ import { SearchResultFile, SearchResultChunk } from "@/features/search/types"; import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweightCodeHighlighter"; import Link from "next/link"; -import { getBrowsePath } from "@/app/[domain]/browse/hooks/useBrowseNavigation"; +import { getBrowsePath } from "@/app/[domain]/browse/hooks/utils"; import { useDomain } from "@/hooks/useDomain"; diff --git a/packages/web/src/app/[domain]/search/page.tsx b/packages/web/src/app/[domain]/search/page.tsx index 2d6e497c..4579e037 100644 --- a/packages/web/src/app/[domain]/search/page.tsx +++ b/packages/web/src/app/[domain]/search/page.tsx @@ -1,378 +1,23 @@ -'use client'; +import { SearchLandingPage } from "./components/searchLandingPage"; +import { SearchResultsPage } from "./components/searchResultsPage"; -import { - ResizablePanel, - ResizablePanelGroup, -} from "@/components/ui/resizable"; -import { Separator } from "@/components/ui/separator"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { useNonEmptyQueryParam } from "@/hooks/useNonEmptyQueryParam"; -import { useSearchHistory } from "@/hooks/useSearchHistory"; -import { SearchQueryParams } from "@/lib/types"; -import { createPathWithQueryParams, measure, unwrapServiceError } from "@/lib/utils"; -import { InfoCircledIcon, SymbolIcon } from "@radix-ui/react-icons"; -import { useQuery } from "@tanstack/react-query"; -import { useRouter } from "next/navigation"; -import { Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { search } from "../../api/(client)/client"; -import { TopBar } from "../components/topBar"; -import { CodePreviewPanel } from "./components/codePreviewPanel"; -import { FilterPanel } from "./components/filterPanel"; -import { SearchResultsPanel } from "./components/searchResultsPanel"; -import { useDomain } from "@/hooks/useDomain"; -import { useToast } from "@/components/hooks/use-toast"; -import { RepositoryInfo, SearchResultFile, SearchStats } from "@/features/search/types"; -import { AnimatedResizableHandle } from "@/components/ui/animatedResizableHandle"; -import { useFilteredMatches } from "./components/filterPanel/useFilterMatches"; -import { Button } from "@/components/ui/button"; -import { ImperativePanelHandle } from "react-resizable-panels"; -import { AlertTriangleIcon, BugIcon, FilterIcon } from "lucide-react"; -import { useHotkeys } from "react-hotkeys-hook"; -import { useLocalStorage } from "@uidotdev/usehooks"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { KeyboardShortcutHint } from "@/app/components/keyboardShortcutHint"; -import { SearchBar } from "../components/searchBar"; -import { CodeSnippet } from "@/app/components/codeSnippet"; -import { CopyIconButton } from "../components/copyIconButton"; - -const DEFAULT_MAX_MATCH_COUNT = 500; - -export default function SearchPage() { - // We need a suspense boundary here since we are accessing query params - // in the top level page. - // @see : https://nextjs.org/docs/messages/missing-suspense-with-csr-bailout - return ( - - - - ) +interface SearchPageProps { + params: Promise<{ domain: string }>; + searchParams: Promise<{ query?: string }>; } -const SearchPageInternal = () => { - const router = useRouter(); - const searchQuery = useNonEmptyQueryParam(SearchQueryParams.query) ?? ""; - const { setSearchHistory } = useSearchHistory(); - const captureEvent = useCaptureEvent(); - const domain = useDomain(); - const { toast } = useToast(); - - // Encodes the number of matches to return in the search response. - const _maxMatchCount = parseInt(useNonEmptyQueryParam(SearchQueryParams.matches) ?? `${DEFAULT_MAX_MATCH_COUNT}`); - const maxMatchCount = isNaN(_maxMatchCount) ? DEFAULT_MAX_MATCH_COUNT : _maxMatchCount; - - const { - data: searchResponse, - isPending: isSearchPending, - isFetching: isFetching, - error - } = useQuery({ - queryKey: ["search", searchQuery, maxMatchCount], - queryFn: () => measure(() => unwrapServiceError(search({ - query: searchQuery, - matches: maxMatchCount, - contextLines: 3, - whole: false, - }, domain)), "client.search"), - select: ({ data, durationMs }) => ({ - ...data, - totalClientSearchDurationMs: durationMs, - }), - enabled: searchQuery.length > 0, - refetchOnWindowFocus: false, - retry: false, - staleTime: 0, - }); - - useEffect(() => { - if (error) { - toast({ - description: `❌ Search failed. Reason: ${error.message}`, - }); - } - }, [error, toast]); - - - // Write the query to the search history - useEffect(() => { - if (searchQuery.length === 0) { - return; - } - - const now = new Date().toUTCString(); - setSearchHistory((searchHistory) => [ - { - query: searchQuery, - date: now, - }, - ...searchHistory.filter(search => search.query !== searchQuery), - ]) - }, [searchQuery, setSearchHistory]); - - useEffect(() => { - if (!searchResponse) { - return; - } - - const fileLanguages = searchResponse.files?.map(file => file.language) || []; - - captureEvent("search_finished", { - durationMs: searchResponse.totalClientSearchDurationMs, - fileCount: searchResponse.stats.fileCount, - matchCount: searchResponse.stats.totalMatchCount, - actualMatchCount: searchResponse.stats.actualMatchCount, - filesSkipped: searchResponse.stats.filesSkipped, - contentBytesLoaded: searchResponse.stats.contentBytesLoaded, - indexBytesLoaded: searchResponse.stats.indexBytesLoaded, - crashes: searchResponse.stats.crashes, - shardFilesConsidered: searchResponse.stats.shardFilesConsidered, - filesConsidered: searchResponse.stats.filesConsidered, - filesLoaded: searchResponse.stats.filesLoaded, - shardsScanned: searchResponse.stats.shardsScanned, - shardsSkipped: searchResponse.stats.shardsSkipped, - shardsSkippedFilter: searchResponse.stats.shardsSkippedFilter, - ngramMatches: searchResponse.stats.ngramMatches, - ngramLookups: searchResponse.stats.ngramLookups, - wait: searchResponse.stats.wait, - matchTreeConstruction: searchResponse.stats.matchTreeConstruction, - matchTreeSearch: searchResponse.stats.matchTreeSearch, - regexpsConsidered: searchResponse.stats.regexpsConsidered, - flushReason: searchResponse.stats.flushReason, - fileLanguages, - }); - }, [captureEvent, searchQuery, searchResponse]); - +export default async function SearchPage(props: SearchPageProps) { + const { domain } = await props.params; + const searchParams = await props.searchParams; + const query = searchParams?.query; - const onLoadMoreResults = useCallback(() => { - const url = createPathWithQueryParams(`/${domain}/search`, - [SearchQueryParams.query, searchQuery], - [SearchQueryParams.matches, `${maxMatchCount * 2}`], - ) - router.push(url); - }, [maxMatchCount, router, searchQuery, domain]); + if (query === undefined || query.length === 0) { + return + } return ( -
- {/* TopBar */} - - - - - {(isSearchPending || isFetching) ? ( -
- -

Searching...

-
- ) : error ? ( -
- -

Failed to search

-

{error.message}

-
- ) : ( - - )} -
- ); -} - -interface PanelGroupProps { - fileMatches: SearchResultFile[]; - isMoreResultsButtonVisible?: boolean; - onLoadMoreResults: () => void; - isBranchFilteringEnabled: boolean; - repoInfo: RepositoryInfo[]; - searchDurationMs: number; - numMatches: number; - searchStats?: SearchStats; -} - -const PanelGroup = ({ - fileMatches, - isMoreResultsButtonVisible, - onLoadMoreResults, - isBranchFilteringEnabled, - repoInfo: _repoInfo, - searchDurationMs: _searchDurationMs, - numMatches, - searchStats, -}: PanelGroupProps) => { - const [previewedFile, setPreviewedFile] = useState(undefined); - const filteredFileMatches = useFilteredMatches(fileMatches); - const filterPanelRef = useRef(null); - const [selectedMatchIndex, setSelectedMatchIndex] = useState(0); - - const [isFilterPanelCollapsed, setIsFilterPanelCollapsed] = useLocalStorage('isFilterPanelCollapsed', false); - - useHotkeys("mod+b", () => { - if (isFilterPanelCollapsed) { - filterPanelRef.current?.expand(); - } else { - filterPanelRef.current?.collapse(); - } - }, { - enableOnFormTags: true, - enableOnContentEditable: true, - description: "Toggle filter panel", - }); - - const searchDurationMs = useMemo(() => { - return Math.round(_searchDurationMs); - }, [_searchDurationMs]); - - const repoInfo = useMemo(() => { - return _repoInfo.reduce((acc, repo) => { - acc[repo.id] = repo; - return acc; - }, {} as Record); - }, [_repoInfo]); - - return ( - - {/* ~~ Filter panel ~~ */} - setIsFilterPanelCollapsed(true)} - onExpand={() => setIsFilterPanelCollapsed(false)} - > - - - {isFilterPanelCollapsed && ( -
- - - - - - - - Open filter panel - - -
- )} - - - {/* ~~ Search results ~~ */} - -
- - - - - -
- -

Search stats for nerds

- { - navigator.clipboard.writeText(JSON.stringify(searchStats, null, 2)); - return true; - }} - className="ml-auto" - /> -
- - {JSON.stringify(searchStats, null, 2)} - - - - { - fileMatches.length > 0 ? ( -

{`[${searchDurationMs} ms] Found ${numMatches} matches in ${fileMatches.length} ${fileMatches.length > 1 ? 'files' : 'file'}`}

- ) : ( -

No results

- ) - } - {isMoreResultsButtonVisible && ( -
- (load more) -
- )} -
- {filteredFileMatches.length > 0 ? ( - { - setSelectedMatchIndex(matchIndex ?? 0); - setPreviewedFile(fileMatch); - }} - isLoadMoreButtonVisible={!!isMoreResultsButtonVisible} - onLoadMoreButtonClicked={onLoadMoreResults} - isBranchFilteringEnabled={isBranchFilteringEnabled} - repoInfo={repoInfo} - /> - ) : ( -
-

No results found

-
- )} - - - {previewedFile && ( - <> - - {/* ~~ Code preview ~~ */} - setPreviewedFile(undefined)} - > - setPreviewedFile(undefined)} - selectedMatchIndex={selectedMatchIndex} - onSelectedMatchIndexChange={setSelectedMatchIndex} - /> - - - )} - + ) } diff --git a/packages/web/src/app/api/(server)/chat/route.ts b/packages/web/src/app/api/(server)/chat/route.ts index 2c9bced2..d7f9368b 100644 --- a/packages/web/src/app/api/(server)/chat/route.ts +++ b/packages/web/src/app/api/(server)/chat/route.ts @@ -1,8 +1,8 @@ import { sew, withAuth, withOrgMembership } from "@/actions"; import { _getConfiguredLanguageModelsFull, _getAISDKLanguageModelAndOptions, updateChatMessages } from "@/features/chat/actions"; import { createAgentStream } from "@/features/chat/agent"; -import { additionalChatRequestParamsSchema, SBChatMessage, SearchScope } from "@/features/chat/types"; -import { getAnswerPartFromAssistantMessage } from "@/features/chat/utils"; +import { additionalChatRequestParamsSchema, LanguageModelInfo, SBChatMessage, SearchScope } from "@/features/chat/types"; +import { getAnswerPartFromAssistantMessage, getLanguageModelKey } from "@/features/chat/utils"; import { ErrorCode } from "@/lib/errorCodes"; import { notFound, schemaValidationError, serviceErrorResponse } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; @@ -49,7 +49,11 @@ export async function POST(req: Request) { return serviceErrorResponse(schemaValidationError(parsed.error)); } - const { messages, id, selectedSearchScopes, languageModelId } = parsed.data; + const { messages, id, selectedSearchScopes, languageModel: _languageModel } = parsed.data; + // @note: a bit of type massaging is required here since the + // zod schema does not enum on `model` or `provider`. + // @see: chat/types.ts + const languageModel = _languageModel as LanguageModelInfo; const response = await sew(() => withAuth((userId) => @@ -78,13 +82,13 @@ export async function POST(req: Request) { // corresponding config in `config.json`. const languageModelConfig = (await _getConfiguredLanguageModelsFull()) - .find((model) => model.model === languageModelId); + .find((model) => getLanguageModelKey(model) === getLanguageModelKey(languageModel)); if (!languageModelConfig) { return serviceErrorResponse({ statusCode: StatusCodes.BAD_REQUEST, errorCode: ErrorCode.INVALID_REQUEST_BODY, - message: `Language model ${languageModelId} is not configured.`, + message: `Language model ${languageModel.model} is not configured.`, }); } diff --git a/packages/web/src/app/globals.css b/packages/web/src/app/globals.css index 6ab78a86..ad3df7af 100644 --- a/packages/web/src/app/globals.css +++ b/packages/web/src/app/globals.css @@ -109,6 +109,7 @@ --chat-citation-border: hsl(217, 91%, 60%); --warning: #ca8a04; + --error: #fc5c5c; } .dark { @@ -201,6 +202,7 @@ --chat-citation-border: hsl(217, 91%, 60%); --warning: #fde047; + --error: #f87171; } } diff --git a/packages/web/src/components/ui/data-table.tsx b/packages/web/src/components/ui/data-table.tsx index ce99592c..26b6417f 100644 --- a/packages/web/src/components/ui/data-table.tsx +++ b/packages/web/src/components/ui/data-table.tsx @@ -62,7 +62,7 @@ export function DataTable({ return (
-
+
({ {headerGroup.headers.map((header) => { return ( - + {header.isPlaceholder ? null : flexRender( @@ -106,7 +109,10 @@ export function DataTable({ data-state={row.getIsSelected() && "selected"} > {row.getVisibleCells().map((cell) => ( - + {flexRender(cell.column.columnDef.cell, cell.getContext())} ))} diff --git a/packages/web/src/components/ui/navigation-menu.tsx b/packages/web/src/components/ui/navigation-menu.tsx index 1419f566..d1dd500b 100644 --- a/packages/web/src/components/ui/navigation-menu.tsx +++ b/packages/web/src/components/ui/navigation-menu.tsx @@ -41,7 +41,7 @@ NavigationMenuList.displayName = NavigationMenuPrimitive.List.displayName const NavigationMenuItem = NavigationMenuPrimitive.Item const navigationMenuTriggerStyle = cva( - "group inline-flex h-10 w-max items-center justify-center rounded-md bg-background px-4 py-2 text-sm font-medium transition-colors hover:bg-accent hover:text-accent-foreground focus:bg-accent focus:text-accent-foreground focus:outline-none disabled:pointer-events-none disabled:opacity-50 data-[active]:bg-accent/50 data-[state=open]:bg-accent/50" + "group inline-flex h-8 w-max items-center justify-center rounded-md bg-background px-1.5 py-1.5 text-sm font-medium transition-colors hover:bg-accent hover:text-accent-foreground focus:bg-accent focus:text-accent-foreground focus:outline-none disabled:pointer-events-none disabled:opacity-50 data-[active]:bg-accent/50 data-[state=open]:bg-accent/50" ) const NavigationMenuTrigger = React.forwardRef< diff --git a/packages/web/src/ee/features/codeNav/components/exploreMenu/referenceList.tsx b/packages/web/src/ee/features/codeNav/components/exploreMenu/referenceList.tsx index 831bebcd..e2febdd2 100644 --- a/packages/web/src/ee/features/codeNav/components/exploreMenu/referenceList.tsx +++ b/packages/web/src/ee/features/codeNav/components/exploreMenu/referenceList.tsx @@ -1,6 +1,6 @@ 'use client'; -import { getBrowsePath } from "@/app/[domain]/browse/hooks/useBrowseNavigation"; +import { getBrowsePath } from "@/app/[domain]/browse/hooks/utils"; import { PathHeader } from "@/app/[domain]/components/pathHeader"; import { LightweightCodeHighlighter } from "@/app/[domain]/components/lightweightCodeHighlighter"; import { FindRelatedSymbolsResponse } from "@/features/codeNav/types"; diff --git a/packages/web/src/env.mjs b/packages/web/src/env.mjs index 7a9c1589..f515c3d9 100644 --- a/packages/web/src/env.mjs +++ b/packages/web/src/env.mjs @@ -146,7 +146,6 @@ export const env = createEnv({ // Misc NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default('unknown'), - NEXT_PUBLIC_POLLING_INTERVAL_MS: numberSchema.default(5000), NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: z.enum(SOURCEBOT_CLOUD_ENVIRONMENT).optional(), @@ -157,7 +156,6 @@ export const env = createEnv({ experimental__runtimeEnv: { NEXT_PUBLIC_POSTHOG_PAPIK: process.env.NEXT_PUBLIC_POSTHOG_PAPIK, NEXT_PUBLIC_SOURCEBOT_VERSION: process.env.NEXT_PUBLIC_SOURCEBOT_VERSION, - NEXT_PUBLIC_POLLING_INTERVAL_MS: process.env.NEXT_PUBLIC_POLLING_INTERVAL_MS, NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT: process.env.NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT, NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY: process.env.NEXT_PUBLIC_LANGFUSE_PUBLIC_KEY, NEXT_PUBLIC_LANGFUSE_BASE_URL: process.env.NEXT_PUBLIC_LANGFUSE_BASE_URL, diff --git a/packages/web/src/features/chat/components/chatBox/atMentionButton.tsx b/packages/web/src/features/chat/components/chatBox/atMentionButton.tsx new file mode 100644 index 00000000..fb17b4ee --- /dev/null +++ b/packages/web/src/features/chat/components/chatBox/atMentionButton.tsx @@ -0,0 +1,38 @@ +'use client'; + +import { Button } from "@/components/ui/button"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { AtSignIcon } from "lucide-react"; +import { useCallback } from "react"; +import { ReactEditor, useSlate } from "slate-react"; +import { AtMentionInfoCard } from "./atMentionInfoCard"; + +// @note: we have this as a seperate component to avoid having to re-render the +// entire toolbar whenever the user types (since we are using the useSlate hook +// here). +export const AtMentionButton = () => { + const editor = useSlate(); + + const onAddContext = useCallback(() => { + editor.insertText("@"); + ReactEditor.focus(editor); + }, [editor]); + + return ( + + + + + + + + + ); +} \ No newline at end of file diff --git a/packages/web/src/features/chat/components/chatBox/chatBox.tsx b/packages/web/src/features/chat/components/chatBox/chatBox.tsx index 934199f2..8876bd69 100644 --- a/packages/web/src/features/chat/components/chatBox/chatBox.tsx +++ b/packages/web/src/features/chat/components/chatBox/chatBox.tsx @@ -27,6 +27,7 @@ interface ChatBoxProps { className?: string; isRedirecting?: boolean; isGenerating?: boolean; + isDisabled?: boolean; languageModels: LanguageModelInfo[]; selectedSearchScopes: SearchScope[]; searchContexts: SearchContextQuery[]; @@ -40,6 +41,7 @@ export const ChatBox = ({ className, isRedirecting, isGenerating, + isDisabled, languageModels, selectedSearchScopes, searchContexts, @@ -68,7 +70,7 @@ export const ChatBox = ({ }).flat(), }); const { selectedLanguageModel } = useSelectedLanguageModel({ - initialLanguageModel: languageModels.length > 0 ? languageModels[0] : undefined, + languageModels, }); const { toast } = useToast(); @@ -167,6 +169,13 @@ export const ChatBox = ({ onContextSelectorOpenChanged(true); } + if (isSubmitDisabledReason === "no-language-model-selected") { + toast({ + description: "⚠️ You must select a language model", + variant: "destructive", + }); + } + return; } @@ -287,6 +296,7 @@ export const ChatBox = ({ renderElement={renderElement} renderLeaf={renderLeaf} onKeyDown={onKeyDown} + readOnly={isDisabled} />
{isRedirecting ? ( diff --git a/packages/web/src/features/chat/components/chatBox/chatBoxToolbar.tsx b/packages/web/src/features/chat/components/chatBox/chatBoxToolbar.tsx index 8bb00cef..a0aae38c 100644 --- a/packages/web/src/features/chat/components/chatBox/chatBoxToolbar.tsx +++ b/packages/web/src/features/chat/components/chatBox/chatBoxToolbar.tsx @@ -1,18 +1,12 @@ 'use client'; -import { Button } from "@/components/ui/button"; import { Separator } from "@/components/ui/separator"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { LanguageModelInfo, SearchScope } from "@/features/chat/types"; import { RepositoryQuery, SearchContextQuery } from "@/lib/types"; -import { AtSignIcon } from "lucide-react"; -import { useCallback } from "react"; -import { ReactEditor, useSlate } from "slate-react"; import { useSelectedLanguageModel } from "../../useSelectedLanguageModel"; +import { AtMentionButton } from "./atMentionButton"; import { LanguageModelSelector } from "./languageModelSelector"; import { SearchScopeSelector } from "./searchScopeSelector"; -import { SearchScopeInfoCard } from "@/features/chat/components/chatBox/searchScopeInfoCard"; -import { AtMentionInfoCard } from "@/features/chat/components/chatBox/atMentionInfoCard"; export interface ChatBoxToolbarProps { languageModels: LanguageModelInfo[]; @@ -33,67 +27,29 @@ export const ChatBoxToolbar = ({ isContextSelectorOpen, onContextSelectorOpenChanged, }: ChatBoxToolbarProps) => { - const editor = useSlate(); - - const onAddContext = useCallback(() => { - editor.insertText("@"); - ReactEditor.focus(editor); - }, [editor]); - const { selectedLanguageModel, setSelectedLanguageModel } = useSelectedLanguageModel({ - initialLanguageModel: languageModels.length > 0 ? languageModels[0] : undefined, + languageModels, }); return ( <> - - - - - - - - + - - - - - - - - - {languageModels.length > 0 && ( - <> - - - -
- -
- - - - )} + + + ) } diff --git a/packages/web/src/features/chat/components/chatBox/languageModelInfoCard.tsx b/packages/web/src/features/chat/components/chatBox/languageModelInfoCard.tsx new file mode 100644 index 00000000..7b24a42d --- /dev/null +++ b/packages/web/src/features/chat/components/chatBox/languageModelInfoCard.tsx @@ -0,0 +1,16 @@ +import { BotIcon } from "lucide-react"; +import Link from "next/link"; + +export const LanguageModelInfoCard = () => { + return ( +
+
+ +

Language Model

+
+
+ Select the language model to use for the chat. Configuration docs. +
+
+ ); +}; \ No newline at end of file diff --git a/packages/web/src/features/chat/components/chatBox/languageModelSelector.tsx b/packages/web/src/features/chat/components/chatBox/languageModelSelector.tsx index 791fa8c8..c8fc0196 100644 --- a/packages/web/src/features/chat/components/chatBox/languageModelSelector.tsx +++ b/packages/web/src/features/chat/components/chatBox/languageModelSelector.tsx @@ -23,6 +23,9 @@ import { } from "lucide-react"; import { useMemo, useState } from "react"; import { ModelProviderLogo } from "./modelProviderLogo"; +import { getLanguageModelKey } from "../../utils"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { LanguageModelInfoCard } from "./languageModelInfoCard"; interface LanguageModelSelectorProps { languageModels: LanguageModelInfo[]; @@ -59,7 +62,7 @@ export const LanguageModelSelector = ({ // De-duplicate models const languageModels = useMemo(() => { return _languageModels.filter((model, selfIndex, selfArray) => - selfIndex === selfArray.findIndex((t) => t.model === model.model) + selfIndex === selfArray.findIndex((t) => getLanguageModelKey(t) === getLanguageModelKey(model)) ); }, [_languageModels]); @@ -68,81 +71,89 @@ export const LanguageModelSelector = ({ open={isPopoverOpen} onOpenChange={setIsPopoverOpen} > - -
- - - setIsPopoverOpen(false)} - > - - - - No models found. - - {languageModels - .map((model, index) => { - const isSelected = selectedModel?.model === model.model; - return ( - { - selectModel(model) - }} - className="cursor-pointer" - > -
- -
- - {model.displayName ?? model.model} - - ); - })} - - - - +
+ {selectedModel ? ( + + ) : ( + + )} + + {selectedModel ? (selectedModel.displayName ?? selectedModel.model) : "Select model"} + + +
+ + + + + + + setIsPopoverOpen(false)} + > + + + + +

No models found.

+ + + {languageModels + .map((model) => { + const isSelected = selectedModel && getLanguageModelKey(selectedModel) === getLanguageModelKey(model); + return ( + { + selectModel(model) + }} + className="cursor-pointer" + > +
+ +
+ + {model.displayName ?? model.model} + + ); + })} + + + + + ); }; diff --git a/packages/web/src/features/chat/components/chatBox/searchScopeSelector.tsx b/packages/web/src/features/chat/components/chatBox/searchScopeSelector.tsx index 6b5b370d..e0b60aea 100644 --- a/packages/web/src/features/chat/components/chatBox/searchScopeSelector.tsx +++ b/packages/web/src/features/chat/components/chatBox/searchScopeSelector.tsx @@ -1,34 +1,29 @@ // Adapted from: web/src/components/ui/multi-select.tsx -import * as React from "react"; -import { - CheckIcon, - ChevronDown, - ScanSearchIcon, -} from "lucide-react"; - -import { cn } from "@/lib/utils"; -import { RepositoryQuery, SearchContextQuery } from "@/lib/types"; -import { Button } from "@/components/ui/button"; import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Separator } from "@/components/ui/separator"; import { Popover, PopoverContent, PopoverTrigger, } from "@/components/ui/popover"; +import { RepositoryQuery, SearchContextQuery } from "@/lib/types"; +import { cn } from "@/lib/utils"; import { - Command, - CommandEmpty, - CommandGroup, - CommandInput, - CommandItem, - CommandList, - CommandSeparator, -} from "@/components/ui/command"; -import { RepoSetSearchScope, RepoSearchScope, SearchScope } from "../../types"; + CheckIcon, + ChevronDown, + ScanSearchIcon, +} from "lucide-react"; +import { ButtonHTMLAttributes, forwardRef, KeyboardEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useVirtualizer } from "@tanstack/react-virtual"; +import { RepoSearchScope, RepoSetSearchScope, SearchScope } from "../../types"; import { SearchScopeIcon } from "../searchScopeIcon"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { SearchScopeInfoCard } from "./searchScopeInfoCard"; -interface SearchScopeSelectorProps extends React.ButtonHTMLAttributes { +interface SearchScopeSelectorProps extends ButtonHTMLAttributes { repos: RepositoryQuery[]; searchContexts: SearchContextQuery[]; selectedSearchScopes: SearchScope[]; @@ -38,7 +33,7 @@ interface SearchScopeSelectorProps extends React.ButtonHTMLAttributes void; } -export const SearchScopeSelector = React.forwardRef< +export const SearchScopeSelector = forwardRef< HTMLButtonElement, SearchScopeSelectorProps >( @@ -55,23 +50,13 @@ export const SearchScopeSelector = React.forwardRef< }, ref ) => { - const scrollContainerRef = React.useRef(null); - const scrollPosition = React.useRef(0); - const [hasSearchInput, setHasSearchInput] = React.useState(false); - - const handleInputKeyDown = ( - event: React.KeyboardEvent - ) => { - if (event.key === "Enter") { - onOpenChanged(true); - } else if (event.key === "Backspace" && !event.currentTarget.value) { - const newSelectedItems = [...selectedSearchScopes]; - newSelectedItems.pop(); - onSelectedSearchScopesChange(newSelectedItems); - } - }; + const scrollContainerRef = useRef(null); + const scrollPosition = useRef(0); + const [searchQuery, setSearchQuery] = useState(""); + const [isMounted, setIsMounted] = useState(false); + const [highlightedIndex, setHighlightedIndex] = useState(0); - const toggleItem = (item: SearchScope) => { + const toggleItem = useCallback((item: SearchScope) => { // Store current scroll position before state update if (scrollContainerRef.current) { scrollPosition.current = scrollContainerRef.current.scrollTop; @@ -88,21 +73,9 @@ export const SearchScopeSelector = React.forwardRef< [...selectedSearchScopes, item]; onSelectedSearchScopesChange(newSelectedItems); - }; - - const handleClear = () => { - onSelectedSearchScopesChange([]); - }; - - const handleSelectAll = () => { - onSelectedSearchScopesChange(allSearchScopeItems); - }; - - const handleTogglePopover = () => { - onOpenChanged(!isOpen); - }; + }, [selectedSearchScopes, onSelectedSearchScopesChange]); - const allSearchScopeItems = React.useMemo(() => { + const allSearchScopeItems = useMemo(() => { const repoSetSearchScopeItems: RepoSetSearchScope[] = searchContexts.map(context => ({ type: 'reposet' as const, value: context.name, @@ -120,8 +93,40 @@ export const SearchScopeSelector = React.forwardRef< return [...repoSetSearchScopeItems, ...repoSearchScopeItems]; }, [repos, searchContexts]); - const sortedSearchScopeItems = React.useMemo(() => { + const handleClear = useCallback(() => { + onSelectedSearchScopesChange([]); + setSearchQuery(""); + requestAnimationFrame(() => { + if (scrollContainerRef.current) { + scrollContainerRef.current.scrollTop = 0; + } + }) + }, [onSelectedSearchScopesChange]); + + const handleSelectAll = useCallback(() => { + onSelectedSearchScopesChange(allSearchScopeItems); + requestAnimationFrame(() => { + if (scrollContainerRef.current) { + scrollContainerRef.current.scrollTop = 0; + } + }); + }, [onSelectedSearchScopesChange, allSearchScopeItems]); + + const handleTogglePopover = useCallback(() => { + onOpenChanged(!isOpen); + }, [onOpenChanged, isOpen]); + + const sortedSearchScopeItems = useMemo(() => { + const query = searchQuery.toLowerCase(); + return allSearchScopeItems + .filter((item) => { + // Filter by search query + if (query && !item.name.toLowerCase().includes(query) && !item.value.toLowerCase().includes(query)) { + return false; + } + return true; + }) .map((item) => ({ item, isSelected: selectedSearchScopes.some( @@ -137,10 +142,77 @@ export const SearchScopeSelector = React.forwardRef< if (a.item.type === 'repo' && b.item.type === 'reposet') return 1; return 0; }) - }, [allSearchScopeItems, selectedSearchScopes]); + }, [allSearchScopeItems, selectedSearchScopes, searchQuery]); + + const handleInputKeyDown = useCallback( + (event: KeyboardEvent) => { + if (event.key === "ArrowDown") { + event.preventDefault(); + setHighlightedIndex((prev) => + prev < sortedSearchScopeItems.length - 1 ? prev + 1 : prev + ); + } else if (event.key === "ArrowUp") { + event.preventDefault(); + setHighlightedIndex((prev) => prev > 0 ? prev - 1 : 0); + } else if (event.key === "Enter") { + event.preventDefault(); + if (sortedSearchScopeItems.length > 0 && highlightedIndex >= 0) { + toggleItem(sortedSearchScopeItems[highlightedIndex].item); + } + } else if (event.key === "Backspace" && !event.currentTarget.value) { + const newSelectedItems = [...selectedSearchScopes]; + newSelectedItems.pop(); + onSelectedSearchScopesChange(newSelectedItems); + } + }, [highlightedIndex, onSelectedSearchScopesChange, selectedSearchScopes, sortedSearchScopeItems, toggleItem]); + + const virtualizer = useVirtualizer({ + count: sortedSearchScopeItems.length, + getScrollElement: () => scrollContainerRef.current, + estimateSize: () => 36, + overscan: 5, + }); + + // Reset highlighted index and scroll to top when search query changes + useEffect(() => { + setHighlightedIndex(0); + if (scrollContainerRef.current) { + scrollContainerRef.current.scrollTop = 0; + } + }, [searchQuery]); + + // Reset highlighted index when items change (but don't scroll) + useEffect(() => { + setHighlightedIndex(0); + }, [sortedSearchScopeItems.length]); + + // Measure virtualizer when popover opens and container is mounted + useEffect(() => { + if (isOpen) { + setIsMounted(true); + setHighlightedIndex(0); + // Give the DOM a tick to render before measuring + requestAnimationFrame(() => { + if (scrollContainerRef.current) { + virtualizer.measure(); + } + }); + } else { + setIsMounted(false); + } + }, [isOpen, virtualizer]); + + // Scroll highlighted item into view + useEffect(() => { + if (isMounted && highlightedIndex >= 0) { + virtualizer.scrollToIndex(highlightedIndex, { + align: 'auto', + }); + } + }, [highlightedIndex, isMounted, virtualizer]); // Restore scroll position after re-render - React.useEffect(() => { + useEffect(() => { if (scrollContainerRef.current && scrollPosition.current > 0) { scrollContainerRef.current.scrollTop = scrollPosition.current; } @@ -151,106 +223,142 @@ export const SearchScopeSelector = React.forwardRef< open={isOpen} onOpenChange={onOpenChanged} > - -
- - - onOpenChanged(false)} - > - - setHasSearchInput(!!value)} - /> - - No results found. - - {!hasSearchInput && ( -
+ + - Select all + { + selectedSearchScopes.length === 0 ? `Search scopes` : + selectedSearchScopes.length === 1 ? selectedSearchScopes[0].name : + `${selectedSearchScopes.length} selected` + } + + +
+ + + + + + + onOpenChanged(false)} + > +
+
+ setSearchQuery(e.target.value)} + onKeyDown={handleInputKeyDown} + className="border-0 focus-visible:ring-0 focus-visible:ring-offset-0 h-11" + /> +
+
+ {sortedSearchScopeItems.length === 0 ? ( +
+ No results found.
- )} - {sortedSearchScopeItems.map(({ item, isSelected }) => { - return ( - toggleItem(item)} - className="cursor-pointer" - > + ) : ( +
+ {!searchQuery && (
- + Select all
-
- -
-
- - {item.name} - - {item.type === 'reposet' && ( - - {item.repoCount} repo{item.repoCount === 1 ? '' : 's'} - + )} +
+ {isMounted && virtualizer.getVirtualItems().map((virtualItem) => { + const { item, isSelected } = sortedSearchScopeItems[virtualItem.index]; + const isHighlighted = virtualItem.index === highlightedIndex; + return ( +
toggleItem(item)} + onMouseEnter={() => setHighlightedIndex(virtualItem.index)} + className={cn( + "cursor-pointer absolute top-0 left-0 w-full flex items-center px-2 py-1.5 text-sm rounded-sm", + isHighlighted ? "bg-accent text-accent-foreground" : "hover:bg-accent hover:text-accent-foreground" )} + style={{ + transform: `translateY(${virtualItem.start}px)`, + }} + > +
+ +
+
+ +
+
+ + {item.name} + + {item.type === 'reposet' && ( + + {item.repoCount} repo{item.repoCount === 1 ? '' : 's'} + + )} +
+
+
-
-
- - ); - })} - - - {selectedSearchScopes.length > 0 && ( - <> - - - Clear - - - )} - - + ); + })} +
+
+ )} +
+ {selectedSearchScopes.length > 0 && ( + <> + +
+ Clear +
+ + )} +
+ + ); } diff --git a/packages/web/src/features/chat/components/chatThread/chatThread.tsx b/packages/web/src/features/chat/components/chatThread/chatThread.tsx index 0c58ae67..e7a49ba9 100644 --- a/packages/web/src/features/chat/components/chatThread/chatThread.tsx +++ b/packages/web/src/features/chat/components/chatThread/chatThread.tsx @@ -25,6 +25,7 @@ import { usePrevious } from '@uidotdev/usehooks'; import { RepositoryQuery, SearchContextQuery } from '@/lib/types'; import { generateAndUpdateChatNameFromMessage } from '../../actions'; import { isServiceError } from '@/lib/utils'; +import { NotConfiguredErrorBanner } from '../notConfiguredErrorBanner'; type ChatHistoryState = { scrollOffset?: number; @@ -73,7 +74,7 @@ export const ChatThread = ({ ); const { selectedLanguageModel } = useSelectedLanguageModel({ - initialLanguageModel: languageModels.length > 0 ? languageModels[0] : undefined, + languageModels, }); const { @@ -118,7 +119,7 @@ export const ChatThread = ({ _sendMessage(message, { body: { selectedSearchScopes, - languageModelId: selectedLanguageModel.model, + languageModel: selectedLanguageModel, } satisfies AdditionalChatRequestParams, }); @@ -355,31 +356,38 @@ export const ChatThread = ({ } {!isChatReadonly && ( -
- - -
- + {languageModels.length === 0 && ( + + )} + +
+ + -
- +
+ +
+ +
)} diff --git a/packages/web/src/features/chat/components/chatThread/tools/shared.tsx b/packages/web/src/features/chat/components/chatThread/tools/shared.tsx index a3e6be86..8274b032 100644 --- a/packages/web/src/features/chat/components/chatThread/tools/shared.tsx +++ b/packages/web/src/features/chat/components/chatThread/tools/shared.tsx @@ -7,7 +7,7 @@ import { cn } from '@/lib/utils'; import { ChevronDown, ChevronRight, Loader2 } from 'lucide-react'; import Link from 'next/link'; import React from 'react'; -import { getBrowsePath } from '@/app/[domain]/browse/hooks/useBrowseNavigation'; +import { getBrowsePath } from "@/app/[domain]/browse/hooks/utils"; export const FileListItem = ({ diff --git a/packages/web/src/features/chat/components/notConfiguredErrorBanner.tsx b/packages/web/src/features/chat/components/notConfiguredErrorBanner.tsx new file mode 100644 index 00000000..3b142bf1 --- /dev/null +++ b/packages/web/src/features/chat/components/notConfiguredErrorBanner.tsx @@ -0,0 +1,18 @@ +import { TriangleAlertIcon } from "lucide-react" +import Link from "next/link" +import { cn } from "@/lib/utils"; + +const DOCS_URL = "https://docs.sourcebot.dev/docs/configuration/language-model-providers"; + +interface NotConfiguredErrorBannerProps { + className?: string; +} + +export const NotConfiguredErrorBanner = ({ className }: NotConfiguredErrorBannerProps) => { + return ( +
+ + Ask unavailable: no language model configured. See the configuration docs for more information. +
+ ) +} \ No newline at end of file diff --git a/packages/web/src/features/chat/types.ts b/packages/web/src/features/chat/types.ts index 524f14f8..8d543b1a 100644 --- a/packages/web/src/features/chat/types.ts +++ b/packages/web/src/features/chat/types.ts @@ -170,6 +170,13 @@ export type LanguageModelProvider = LanguageModel['provider']; // This is a subset of information about a configured // language model that we can safely send to the client. +// @note: ensure this is in sync with the LanguageModelInfo type. +export const languageModelInfoSchema = z.object({ + provider: z.string(), + model: z.string(), + displayName: z.string().optional(), +}); + export type LanguageModelInfo = { provider: LanguageModelProvider, model: LanguageModel['model'], @@ -178,7 +185,7 @@ export type LanguageModelInfo = { // Additional request body data that we send along to the chat API. export const additionalChatRequestParamsSchema = z.object({ - languageModelId: z.string(), + languageModel: languageModelInfoSchema, selectedSearchScopes: z.array(searchScopeSchema), }); export type AdditionalChatRequestParams = z.infer; \ No newline at end of file diff --git a/packages/web/src/features/chat/useSelectedLanguageModel.ts b/packages/web/src/features/chat/useSelectedLanguageModel.ts index 7cdc79e4..a22b5940 100644 --- a/packages/web/src/features/chat/useSelectedLanguageModel.ts +++ b/packages/web/src/features/chat/useSelectedLanguageModel.ts @@ -2,22 +2,40 @@ import { useLocalStorage } from "usehooks-ts"; import { LanguageModelInfo } from "./types"; +import { useEffect } from "react"; +import { getLanguageModelKey } from "./utils"; type Props = { - initialLanguageModel?: LanguageModelInfo; + languageModels: LanguageModelInfo[]; } export const useSelectedLanguageModel = ({ - initialLanguageModel, -}: Props = {}) => { + languageModels, +}: Props) => { + const fallbackLanguageModel = languageModels.length > 0 ? languageModels[0] : undefined; const [selectedLanguageModel, setSelectedLanguageModel] = useLocalStorage( "selectedLanguageModel", - initialLanguageModel, + fallbackLanguageModel, { initializeWithValue: false, } ); + // Handle the case where the selected language model is no longer + // available. Reset to the fallback language model in this case. + useEffect(() => { + if (!selectedLanguageModel || !languageModels.find( + (model) => getLanguageModelKey(model) === getLanguageModelKey(selectedLanguageModel) + )) { + setSelectedLanguageModel(fallbackLanguageModel); + } + }, [ + fallbackLanguageModel, + languageModels, + selectedLanguageModel, + setSelectedLanguageModel, + ]); + return { selectedLanguageModel, setSelectedLanguageModel, diff --git a/packages/web/src/features/chat/utils.ts b/packages/web/src/features/chat/utils.ts index c57f3fd5..f339b872 100644 --- a/packages/web/src/features/chat/utils.ts +++ b/packages/web/src/features/chat/utils.ts @@ -6,6 +6,7 @@ import { CustomText, FileReference, FileSource, + LanguageModelInfo, MentionData, MentionElement, ParagraphElement, @@ -365,4 +366,11 @@ export const buildSearchQuery = (options: { } return query; -} \ No newline at end of file +} + +/** + * Generates a unique key given a LanguageModelInfo object. + */ +export const getLanguageModelKey = (model: LanguageModelInfo) => { + return `${model.provider}-${model.model}-${model.displayName}`; +} diff --git a/packages/web/src/features/fileTree/components/pureFileTreePanel.tsx b/packages/web/src/features/fileTree/components/pureFileTreePanel.tsx index 88c1ab2f..77b4622a 100644 --- a/packages/web/src/features/fileTree/components/pureFileTreePanel.tsx +++ b/packages/web/src/features/fileTree/components/pureFileTreePanel.tsx @@ -4,7 +4,7 @@ import { FileTreeNode as RawFileTreeNode } from "../actions"; import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area"; import React, { useCallback, useMemo, useState, useEffect, useRef } from "react"; import { FileTreeItemComponent } from "./fileTreeItemComponent"; -import { getBrowsePath } from "@/app/[domain]/browse/hooks/useBrowseNavigation"; +import { getBrowsePath } from "@/app/[domain]/browse/hooks/utils"; import { useBrowseParams } from "@/app/[domain]/browse/hooks/useBrowseParams"; import { useDomain } from "@/hooks/useDomain"; @@ -116,7 +116,7 @@ export const PureFileTreePanel = ({ tree: _tree, path }: PureFileTreePanelProps) })} ); - }, [path]); + }, [domain, path, repoName, revisionName, setIsCollapsed]); const renderedTree = useMemo(() => renderTree(tree), [tree, renderTree]); diff --git a/packages/web/src/initialize.ts b/packages/web/src/initialize.ts index 4c7b0355..1063e58c 100644 --- a/packages/web/src/initialize.ts +++ b/packages/web/src/initialize.ts @@ -1,4 +1,4 @@ -import { ConnectionSyncStatus, OrgRole, Prisma, RepoIndexingStatus } from '@sourcebot/db'; +import { ConnectionSyncStatus, OrgRole, Prisma } from '@sourcebot/db'; import { env } from './env.mjs'; import { prisma } from "@/prisma"; import { SINGLE_TENANT_ORG_ID, SINGLE_TENANT_ORG_DOMAIN, SOURCEBOT_GUEST_USER_ID, SINGLE_TENANT_ORG_NAME } from './lib/constants'; @@ -64,21 +64,6 @@ const syncConnections = async (connections?: { [key: string]: ConnectionConfig } }); logger.info(`Upserted connection with name '${key}'. Connection ID: ${connectionDb.id}`); - - // Re-try any repos that failed to index. - const failedRepos = currentConnection?.repos.filter(repo => repo.repo.repoIndexingStatus === RepoIndexingStatus.FAILED).map(repo => repo.repo.id) ?? []; - if (failedRepos.length > 0) { - await prisma.repo.updateMany({ - where: { - id: { - in: failedRepos, - } - }, - data: { - repoIndexingStatus: RepoIndexingStatus.NEW, - } - }) - } } } diff --git a/packages/web/src/lib/constants.ts b/packages/web/src/lib/constants.ts index e74e7d51..3cd6e258 100644 --- a/packages/web/src/lib/constants.ts +++ b/packages/web/src/lib/constants.ts @@ -23,7 +23,6 @@ export const TEAM_FEATURES = [ ] export const MOBILE_UNSUPPORTED_SPLASH_SCREEN_DISMISSED_COOKIE_NAME = 'sb.mobile-unsupported-splash-screen-dismissed'; -export const SEARCH_MODE_COOKIE_NAME = 'sb.search-mode'; export const AGENTIC_SEARCH_TUTORIAL_DISMISSED_COOKIE_NAME = 'sb.agentic-search-tutorial-dismissed'; // NOTE: changing SOURCEBOT_GUEST_USER_ID may break backwards compatibility since this value is used diff --git a/packages/web/src/lib/schemas.ts b/packages/web/src/lib/schemas.ts index 7d37d9ca..fa8fbc7a 100644 --- a/packages/web/src/lib/schemas.ts +++ b/packages/web/src/lib/schemas.ts @@ -1,5 +1,4 @@ import { checkIfOrgDomainExists } from "@/actions"; -import { RepoIndexingStatus } from "@sourcebot/db"; import { z } from "zod"; import { isServiceError } from "./utils"; import { serviceErrorSchema } from "./serviceError"; @@ -22,7 +21,6 @@ export const repositoryQuerySchema = z.object({ webUrl: z.string().optional(), imageUrl: z.string().optional(), indexedAt: z.coerce.date().optional(), - repoIndexingStatus: z.nativeEnum(RepoIndexingStatus), }); export const searchContextQuerySchema = z.object({ diff --git a/packages/web/src/lib/utils.ts b/packages/web/src/lib/utils.ts index 38b52cfe..b35de40b 100644 --- a/packages/web/src/lib/utils.ts +++ b/packages/web/src/lib/utils.ts @@ -376,6 +376,19 @@ export const getDisplayTime = (date: Date) => { } } +/** + * Converts a number to a string + */ +export const getShortenedNumberDisplayString = (number: number) => { + if (number < 1000) { + return number.toString(); + } else if (number < 1000000) { + return `${(number / 1000).toFixed(1)}k`; + } else { + return `${(number / 1000000).toFixed(1)}m`; + } +} + export const measureSync = (cb: () => T, measureName: string, outputLog: boolean = true) => { const startMark = `${measureName}.start`; const endMark = `${measureName}.end`; diff --git a/packages/web/src/prisma.ts b/packages/web/src/prisma.ts index 1d4b7585..860d23fa 100644 --- a/packages/web/src/prisma.ts +++ b/packages/web/src/prisma.ts @@ -57,4 +57,4 @@ export const userScopedPrismaClientExtension = (userId?: string) => { } }) }) -} \ No newline at end of file +} diff --git a/packages/web/tailwind.config.ts b/packages/web/tailwind.config.ts index f04b5b77..c2859825 100644 --- a/packages/web/tailwind.config.ts +++ b/packages/web/tailwind.config.ts @@ -67,6 +67,7 @@ const config = { ring: 'var(--sidebar-ring)' }, warning: 'var(--warning)', + error: 'var(--error)', editor: { background: 'var(--editor-background)', foreground: 'var(--editor-foreground)', @@ -149,7 +150,8 @@ const config = { 'accordion-down': 'accordion-down 0.2s ease-out', 'accordion-up': 'accordion-up 0.2s ease-out', 'spin-slow': 'spin 1.5s linear infinite', - 'bounce-slow': 'bounce 1.5s linear infinite' + 'bounce-slow': 'bounce 1.5s linear infinite', + 'ping-slow': 'ping 1.5s cubic-bezier(0, 0, 0.2, 1) infinite' } } }, diff --git a/yarn.lock b/yarn.lock index 7fae191b..8cffa0aa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7571,6 +7571,7 @@ __metadata: git-url-parse: "npm:^16.1.0" gitea-js: "npm:^1.22.0" glob: "npm:^11.0.0" + groupmq: "npm:^1.0.0" ioredis: "npm:^5.4.2" json-schema-to-typescript: "npm:^15.0.4" lowdb: "npm:^7.0.1" @@ -10026,6 +10027,17 @@ __metadata: languageName: node linkType: hard +"cliui@npm:^8.0.1": + version: 8.0.1 + resolution: "cliui@npm:8.0.1" + dependencies: + string-width: "npm:^4.2.0" + strip-ansi: "npm:^6.0.1" + wrap-ansi: "npm:^7.0.0" + checksum: 10c0/4bda0f09c340cbb6dfdc1ed508b3ca080f12992c18d68c6be4d9cf51756033d5266e61ec57529e610dacbf4da1c634423b0c1b11037709cc6b09045cbd815df5 + languageName: node + linkType: hard + "clone@npm:^1.0.2": version: 1.0.4 resolution: "clone@npm:1.0.4" @@ -10424,6 +10436,23 @@ __metadata: languageName: node linkType: hard +"concurrently@npm:^9.2.1": + version: 9.2.1 + resolution: "concurrently@npm:9.2.1" + dependencies: + chalk: "npm:4.1.2" + rxjs: "npm:7.8.2" + shell-quote: "npm:1.8.3" + supports-color: "npm:8.1.1" + tree-kill: "npm:1.2.2" + yargs: "npm:17.7.2" + bin: + conc: dist/bin/concurrently.js + concurrently: dist/bin/concurrently.js + checksum: 10c0/da37f239f82eb7ac24f5ddb56259861e5f1d6da2ade7602b6ea7ad3101b13b5ccec02a77b7001402d1028ff2fdc38eed55644b32853ad5abf30e057002a963aa + languageName: node + linkType: hard + "content-disposition@npm:0.5.4": version: 0.5.4 resolution: "content-disposition@npm:0.5.4" @@ -11764,7 +11793,7 @@ __metadata: languageName: node linkType: hard -"escalade@npm:^3.2.0": +"escalade@npm:^3.1.1, escalade@npm:^3.2.0": version: 3.2.0 resolution: "escalade@npm:3.2.0" checksum: 10c0/ced4dd3a78e15897ed3be74e635110bbf3b08877b0a41be50dcb325ee0e0b5f65fc2d50e9845194d7c4633f327e2e1c6cce00a71b617c5673df0374201d67f65 @@ -12726,6 +12755,13 @@ __metadata: languageName: node linkType: hard +"get-caller-file@npm:^2.0.5": + version: 2.0.5 + resolution: "get-caller-file@npm:2.0.5" + checksum: 10c0/c6c7b60271931fa752aeb92f2b47e355eac1af3a2673f47c9589e8f8a41adc74d45551c1bc57b5e66a80609f10ffb72b6f575e4370d61cc3f7f3aaff01757cde + languageName: node + linkType: hard + "get-intrinsic@npm:^1.2.4, get-intrinsic@npm:^1.2.5, get-intrinsic@npm:^1.2.6, get-intrinsic@npm:^1.2.7, get-intrinsic@npm:^1.3.0": version: 1.3.0 resolution: "get-intrinsic@npm:1.3.0" @@ -13024,6 +13060,17 @@ __metadata: languageName: node linkType: hard +"groupmq@npm:^1.0.0": + version: 1.0.0 + resolution: "groupmq@npm:1.0.0" + dependencies: + cron-parser: "npm:^4.9.0" + peerDependencies: + ioredis: ">=5" + checksum: 10c0/1795e483fc712ff91a2f7abb32f29f14f94b9d5e32033df064021f284a12773822e241f0d5e061978632dcf20df5db3f3d7701f04e53979d5b594f618fc53a27 + languageName: node + linkType: hard + "gtoken@npm:^7.0.0": version: 7.1.0 resolution: "gtoken@npm:7.1.0" @@ -17520,6 +17567,13 @@ __metadata: languageName: node linkType: hard +"require-directory@npm:^2.1.1": + version: 2.1.1 + resolution: "require-directory@npm:2.1.1" + checksum: 10c0/83aa76a7bc1531f68d92c75a2ca2f54f1b01463cb566cf3fbc787d0de8be30c9dbc211d1d46be3497dac5785fe296f2dd11d531945ac29730643357978966e99 + languageName: node + linkType: hard + "require-from-string@npm:^2.0.2": version: 2.0.2 resolution: "require-from-string@npm:2.0.2" @@ -17834,9 +17888,9 @@ __metadata: version: 0.0.0-use.local resolution: "root-workspace-0b6124@workspace:." dependencies: + concurrently: "npm:^9.2.1" cross-env: "npm:^7.0.3" dotenv-cli: "npm:^8.0.0" - npm-run-all: "npm:^4.1.5" languageName: unknown linkType: soft @@ -17918,6 +17972,15 @@ __metadata: languageName: node linkType: hard +"rxjs@npm:7.8.2": + version: 7.8.2 + resolution: "rxjs@npm:7.8.2" + dependencies: + tslib: "npm:^2.1.0" + checksum: 10c0/1fcd33d2066ada98ba8f21fcbbcaee9f0b271de1d38dc7f4e256bfbc6ffcdde68c8bfb69093de7eeb46f24b1fb820620bf0223706cff26b4ab99a7ff7b2e2c45 + languageName: node + linkType: hard + "safe-array-concat@npm:^1.1.3": version: 1.1.3 resolution: "safe-array-concat@npm:1.1.3" @@ -18346,6 +18409,13 @@ __metadata: languageName: node linkType: hard +"shell-quote@npm:1.8.3": + version: 1.8.3 + resolution: "shell-quote@npm:1.8.3" + checksum: 10c0/bee87c34e1e986cfb4c30846b8e6327d18874f10b535699866f368ade11ea4ee45433d97bf5eada22c4320c27df79c3a6a7eb1bf3ecfc47f2c997d9e5e2672fd + languageName: node + linkType: hard + "shell-quote@npm:^1.6.1": version: 1.8.2 resolution: "shell-quote@npm:1.8.2" @@ -18767,7 +18837,7 @@ __metadata: languageName: node linkType: hard -"string-width-cjs@npm:string-width@^4.2.0, string-width@npm:^4.1.0": +"string-width-cjs@npm:string-width@^4.2.0, string-width@npm:^4.1.0, string-width@npm:^4.2.0, string-width@npm:^4.2.3": version: 4.2.3 resolution: "string-width@npm:4.2.3" dependencies: @@ -19031,6 +19101,15 @@ __metadata: languageName: node linkType: hard +"supports-color@npm:8.1.1": + version: 8.1.1 + resolution: "supports-color@npm:8.1.1" + dependencies: + has-flag: "npm:^4.0.0" + checksum: 10c0/ea1d3c275dd604c974670f63943ed9bd83623edc102430c05adb8efc56ba492746b6e95386e7831b872ec3807fd89dd8eb43f735195f37b5ec343e4234cc7e89 + languageName: node + linkType: hard + "supports-color@npm:^5.3.0, supports-color@npm:^5.5.0": version: 5.5.0 resolution: "supports-color@npm:5.5.0" @@ -19341,6 +19420,15 @@ __metadata: languageName: node linkType: hard +"tree-kill@npm:1.2.2": + version: 1.2.2 + resolution: "tree-kill@npm:1.2.2" + bin: + tree-kill: cli.js + checksum: 10c0/7b1b7c7f17608a8f8d20a162e7957ac1ef6cd1636db1aba92f4e072dc31818c2ff0efac1e3d91064ede67ed5dc57c565420531a8134090a12ac10cf792ab14d2 + languageName: node + linkType: hard + "trim-lines@npm:^3.0.0": version: 3.0.1 resolution: "trim-lines@npm:3.0.1" @@ -20390,7 +20478,7 @@ __metadata: languageName: node linkType: hard -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0, wrap-ansi@npm:^7.0.0": version: 7.0.0 resolution: "wrap-ansi@npm:7.0.0" dependencies: @@ -20477,6 +20565,13 @@ __metadata: languageName: node linkType: hard +"y18n@npm:^5.0.5": + version: 5.0.8 + resolution: "y18n@npm:5.0.8" + checksum: 10c0/4df2842c36e468590c3691c894bc9cdbac41f520566e76e24f59401ba7d8b4811eb1e34524d57e54bc6d864bcb66baab7ffd9ca42bf1eda596618f9162b91249 + languageName: node + linkType: hard + "yallist@npm:^3.0.2": version: 3.1.1 resolution: "yallist@npm:3.1.1" @@ -20507,6 +20602,28 @@ __metadata: languageName: node linkType: hard +"yargs-parser@npm:^21.1.1": + version: 21.1.1 + resolution: "yargs-parser@npm:21.1.1" + checksum: 10c0/f84b5e48169479d2f402239c59f084cfd1c3acc197a05c59b98bab067452e6b3ea46d4dd8ba2985ba7b3d32a343d77df0debd6b343e5dae3da2aab2cdf5886b2 + languageName: node + linkType: hard + +"yargs@npm:17.7.2": + version: 17.7.2 + resolution: "yargs@npm:17.7.2" + dependencies: + cliui: "npm:^8.0.1" + escalade: "npm:^3.1.1" + get-caller-file: "npm:^2.0.5" + require-directory: "npm:^2.1.1" + string-width: "npm:^4.2.3" + y18n: "npm:^5.0.5" + yargs-parser: "npm:^21.1.1" + checksum: 10c0/ccd7e723e61ad5965fffbb791366db689572b80cca80e0f96aad968dfff4156cd7cd1ad18607afe1046d8241e6fb2d6c08bf7fa7bfb5eaec818735d8feac8f05 + languageName: node + linkType: hard + "yocto-queue@npm:^0.1.0": version: 0.1.0 resolution: "yocto-queue@npm:0.1.0"