add review command logic and add logging for review agent to data cache dir

Author: msukkari

.env.development

@@ -23,6 +23,8 @@ AUTH_URL="http://localhost:3000"
 # AUTH_GOOGLE_CLIENT_ID=""
 # AUTH_GOOGLE_CLIENT_SECRET=""
 
+#DATA_CACHE_DIR="" # Path to the sourcebot cache dir (ex. ~/sourcebot/.sourcebot)
+
 # Email
 # EMAIL_FROM_ADDRESS="" # The from address for transactional emails.
 # SMTP_CONNECTION_URL="" # The SMTP connection URL for transactional emails.
@@ -51,6 +53,16 @@ REDIS_URL="redis://localhost:6379"
 # STRIPE_WEBHOOK_SECRET: z.string().optional(),
 # STRIPE_ENABLE_TEST_CLOCKS=false
 
+# Agents
+
+# GITHUB_APP_ID=
+# GITHUB_APP_PRIVATE_KEY_PATH=
+# GITHUB_APP_WEBHOOK_SECRET=
+# OPENAI_API_KEY=
+REVIEW_AGENT_LOGGING_ENABLED=true
+REVIEW_AGENT_AUTO_REVIEW_ENABLED=false
+REVIEW_AGENT_REVIEW_COMMAND=review
+
 # Misc
 
 # Generated using:
@@ -79,9 +91,3 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
 # SOURCEBOT_TENANCY_MODE=single
 
 # NEXT_PUBLIC_SOURCEBOT_CLOUD_ENVIRONMENT=
-
-# Used for agents
-# GITHUB_APP_ID=
-# GITHUB_APP_PRIVATE_KEY_PATH=
-# GITHUB_APP_WEBHOOK_SECRET=
-# OPENAI_API_KEY=

docs/docs/agents/review-agent.mdx

@@ -30,10 +30,12 @@ Before you get started, make sure you have an OpenAPI account that you can creat
         - Webhook URL (**IMPORTANT**): You must set this to point to your Sourcebot deployment at /api/webhook (ex. https://sourcebot.aperture.com/api/webhook). Your Sourcebot deployment must be able to accept requests from GitHub
         (either github.com or your self-hosted enterprise server) for this to work. If you're running Sourcebot locally, you can [use smee](https://docs.github.com/en/apps/creating-github-apps/writing-code-for-a-github-app/quickstart#step-2-get-a-webhook-proxy-url) to [forward webhooks](https://docs.github.com/en/apps/creating-github-apps/writing-code-for-a-github-app/quickstart#step-6-start-your-server) to you local deployment.
         - Permissions
-            - Pull requests: Read
+            - Pull requests: Read & Write
+            - Issues: Read & Write
             - Contents: Read
         - Events:
             - Pull request
+            - Issue comment
     </Step>
     <Step title="Install the GitHub app in your organization">
         Navigate to your new GitHub app's page and press `Install`. You can find this in your [app settings](https://docs.github.com/en/apps/creating-github-apps/writing-code-for-a-github-app/quickstart#navigate-to-your-app-settings). 
@@ -48,10 +50,19 @@ Before you get started, make sure you have an OpenAPI account that you can creat
         - `GITHUB_APP_PRIVATE_KEY_PATH`: The path to your app's private key. You can generate a private key file for your app in the [app settings](https://docs.github.com/en/apps/creating-github-apps/writing-code-for-a-github-app/quickstart#navigate-to-your-app-settings)
         ![GitHub App Private Key](/images/github_app_private_key.png)
         - `OPENAI_API_KEY`: Your OpenAI API key
+        - `REVIEW_AGENT_AUTO_REVIEW_ENABLED` (default: `false`): If enabled, the review agent will automatically review any new or updated PR. If disabled, you must invoke it using the command defined by `REVIEW_AGENT_REVIEW_COMMAND`
+        - `REVIEW_AGENT_REVIEW_COMMAND` (default: `review`): The command that invokes the review agent (ex. `/review`) when a user comments on the PR. Don't include the slash character in this value.
     </Step>
     <Step title="Verify configuration">
-        Navigate to the agents page by pressing `Agents` in the Sourcebot nav menu. If you've configured your environment variables you'll see the following:
+        Navigate to the agents page by pressing `Agents` in the Sourcebot nav menu. If you've configured your environment variables correctly you'll see the following:
 
         ![Review Agent Configured](/images/review_agent_configured.png)
     </Step>
-</Steps>
+</Steps>
+
+# Using the agent
+
+The review agent will not automatically review your PRs by default. To enable this feature, set the `REVIEW_AGENT_AUTO_REVIEW_ENABLED` environment variable to true.
+
+You can invoke the review agent manually by commenting `/review` on the PR you'd like it to review. You can configure the command that triggers the agent by changing
+the `REVIEW_AGENT_REVIEW_COMMAND` environment variable. 

packages/backend/src/env.ts

@@ -27,6 +27,8 @@ export const env = createEnv({
         SOURCEBOT_INSTALL_ID: z.string().default("unknown"),
         NEXT_PUBLIC_SOURCEBOT_VERSION: z.string().default("unknown"),
 
+        DATA_CACHE_DIR: z.string(),
+
         NEXT_PUBLIC_POSTHOG_PAPIK: z.string().optional(),
 
         FALLBACK_GITHUB_CLOUD_TOKEN: z.string().optional(),

packages/backend/src/index.ts

@@ -8,6 +8,7 @@ import path from 'path';
 import { AppContext } from "./types.js";
 import { main } from "./main.js"
 import { PrismaClient } from "@sourcebot/db";
+import { env } from "./env.js";
 
 // Register handler for normal exit
 process.on('exit', (code) => {
@@ -36,22 +37,9 @@ process.on('unhandledRejection', (reason, promise) => {
     process.exit(1);
 });
 
+console.log(process.cwd());
 
-const parser = new ArgumentParser({
-    description: "Sourcebot backend tool",
-});
-
-type Arguments = {
-    cacheDir: string;
-}
-
-parser.add_argument("--cacheDir", {
-    help: "Path to .sourcebot cache directory",
-    required: true,
-});
-const args = parser.parse_args() as Arguments;
-
-const cacheDir = args.cacheDir;
+const cacheDir = env.DATA_CACHE_DIR;
 const reposPath = path.join(cacheDir, 'repos');
 const indexPath = path.join(cacheDir, 'index');
 

packages/web/src/app/api/(server)/webhook/route.ts

@@ -8,6 +8,7 @@ import { env } from "@/env.mjs";
 import { processGitHubPullRequest } from "@/features/agents/review-agent/app";
 import { throttling } from "@octokit/plugin-throttling";
 import fs from "fs";
+import { GitHubPullRequest } from "@/features/agents/review-agent/types";
 
 let githubApp: App | undefined;
 if (env.GITHUB_APP_ID && env.GITHUB_APP_WEBHOOK_SECRET && env.GITHUB_APP_PRIVATE_KEY_PATH) {
@@ -42,6 +43,10 @@ function isPullRequestEvent(eventHeader: string, payload: unknown): payload is W
     return eventHeader === "pull_request" && typeof payload === "object" && payload !== null && "action" in payload && typeof payload.action === "string" && (payload.action === "opened" || payload.action === "synchronize");
 }
 
+function isIssueCommentEvent(eventHeader: string, payload: unknown): payload is WebhookEventDefinition<"issue-comment-created"> {
+    return eventHeader === "issue_comment" && typeof payload === "object" && payload !== null && "action" in payload && typeof payload.action === "string" && payload.action === "created";
+}
+
 export const POST = async (request: NextRequest) => {
     const body = await request.json();
     const headers = Object.fromEntries(request.headers.entries());
@@ -56,6 +61,11 @@ export const POST = async (request: NextRequest) => {
         }
 
         if (isPullRequestEvent(githubEvent, body)) {
+            if (env.REVIEW_AGENT_AUTO_REVIEW_ENABLED === "false") {
+                console.log('Review agent auto review (REVIEW_AGENT_AUTO_REVIEW_ENABLED) is disabled, skipping');
+                return Response.json({ status: 'ok' });
+            }
+
             if (!body.installation) {
                 console.error('Received github pull request event but installation is not present');
                 return Response.json({ status: 'ok' });
@@ -64,7 +74,38 @@ export const POST = async (request: NextRequest) => {
             const installationId = body.installation.id;
             const octokit = await githubApp.getInstallationOctokit(installationId);
 
-            await processGitHubPullRequest(octokit, body);
+            const pullRequest = body.pull_request as GitHubPullRequest;
+            await processGitHubPullRequest(octokit, pullRequest);
+        }
+
+        if (isIssueCommentEvent(githubEvent, body)) {
+            const comment = body.comment.body;
+            if (!comment) {
+                console.warn('Received issue comment event but comment body is empty');
+                return Response.json({ status: 'ok' });
+            }
+
+            if (comment === `/${env.REVIEW_AGENT_REVIEW_COMMAND}`) {
+                console.log('Review agent review command received, processing');
+
+                if (!body.installation) {
+                    console.error('Received github issue comment event but installation is not present');
+                    return Response.json({ status: 'ok' });
+                }
+
+                const pullRequestNumber = body.issue.number;
+                const repositoryName = body.repository.name;
+                const owner = body.repository.owner.login;
+
+                const octokit = await githubApp.getInstallationOctokit(body.installation.id);
+                const { data: pullRequest } = await octokit.rest.pulls.get({
+                    owner,
+                    repo: repositoryName,
+                    pull_number: pullRequestNumber,
+                });
+
+                await processGitHubPullRequest(octokit, pullRequest);
+            }
         }
     }
 

packages/web/src/env.mjs

@@ -27,6 +27,8 @@ export const env = createEnv({
         AUTH_URL: z.string().url(),
         AUTH_CREDENTIALS_LOGIN_ENABLED: booleanSchema.default('true'),
 
+        DATA_CACHE_DIR: z.string(),
+
         // Email
         SMTP_CONNECTION_URL: z.string().url().optional(),
         EMAIL_FROM_ADDRESS: z.string().email().optional(),
@@ -58,6 +60,9 @@ export const env = createEnv({
         GITHUB_APP_WEBHOOK_SECRET: z.string().optional(),
         GITHUB_APP_PRIVATE_KEY_PATH: z.string().optional(),
         OPENAI_API_KEY: z.string().optional(),
+        REVIEW_AGENT_LOGGING_ENABLED: booleanSchema.default('true'),
+        REVIEW_AGENT_AUTO_REVIEW_ENABLED: booleanSchema.default('false'),
+        REVIEW_AGENT_REVIEW_COMMAND: z.string().default('review'),
     },
     // @NOTE: Please make sure of the following:
     // - Make sure you destructure all client variables in

packages/web/src/features/agents/review-agent/app.ts

@@ -1,9 +1,11 @@
 import { Octokit } from "octokit";
-import { WebhookEventDefinition } from "@octokit/webhooks/types";
 import { generatePrReviews } from "@/features/agents/review-agent/nodes/generatePrReview";
 import { githubPushPrReviews } from "@/features/agents/review-agent/nodes/githubPushPrReviews";
 import { githubPrParser } from "@/features/agents/review-agent/nodes/githubPrParser";
 import { env } from "@/env.mjs";
+import { GitHubPullRequest } from "@/features/agents/review-agent/types";
+import path from "path";
+import fs from "fs";
 
 const rules = [
     "Do NOT provide general feedback, summaries, explanations of changes, or praises for making good additions.",
@@ -15,15 +17,27 @@ const rules = [
     "If there are no issues found on a line range, do NOT respond with any comments. This includes comments such as \"No issues found\" or \"LGTM\"."
 ]
 
-export async function processGitHubPullRequest(octokit: Octokit, payload: WebhookEventDefinition<"pull-request-opened"> | WebhookEventDefinition<"pull-request-synchronize">) {
-    console.log(`Received a pull request event for #${payload.pull_request.number}`);
+export async function processGitHubPullRequest(octokit: Octokit, pullRequest: GitHubPullRequest) {
+    console.log(`Received a pull request event for #${pullRequest.number}`);
 
     if (!env.OPENAI_API_KEY) {
         console.error("OPENAI_API_KEY is not set, skipping review agent");
         return;
     }
 
-    const prPayload = await githubPrParser(octokit, payload);
-    const fileDiffReviews = await generatePrReviews(prPayload, rules);
+    let reviewAgentLogPath: string | undefined;
+    if (env.REVIEW_AGENT_LOGGING_ENABLED) {
+        const reviewAgentLogDir = path.join(env.DATA_CACHE_DIR, "review-agent");
+        if (!fs.existsSync(reviewAgentLogDir)) {
+            fs.mkdirSync(reviewAgentLogDir, { recursive: true });
+        }
+
+        const timestamp = new Date().toLocaleString().replace(/[/: ,]/g, '');
+        reviewAgentLogPath = path.join(reviewAgentLogDir, `review-agent-${pullRequest.number}-${timestamp}.log`);
+        console.log(`Review agent logging to ${reviewAgentLogPath}`);
+    }
+
+    const prPayload = await githubPrParser(octokit, pullRequest);
+    const fileDiffReviews = await generatePrReviews(reviewAgentLogPath, prPayload, rules);
     await githubPushPrReviews(octokit, prPayload, fileDiffReviews); 
 }

packages/web/src/features/agents/review-agent/nodes/generatePrReview.ts

@@ -3,7 +3,7 @@ import { generateDiffReviewPrompt } from "@/features/agents/review-agent/nodes/g
 import { invokeDiffReviewLlm } from "@/features/agents/review-agent/nodes/invokeDiffReviewLlm";
 import { fetchFileContent } from "@/features/agents/review-agent/nodes/fetchFileContent";
 
-export const generatePrReviews = async (pr_payload: sourcebot_pr_payload, rules: string[]): Promise<sourcebot_file_diff_review[]> => {
+export const generatePrReviews = async (reviewAgentLogPath: string | undefined, pr_payload: sourcebot_pr_payload, rules: string[]): Promise<sourcebot_file_diff_review[]> => {
     console.log("Executing generate_pr_reviews");
 
     const file_diff_reviews: sourcebot_file_diff_review[] = [];
@@ -29,7 +29,7 @@ export const generatePrReviews = async (pr_payload: sourcebot_pr_payload, rules:
 
                 const prompt = await generateDiffReviewPrompt(diff, context, rules);
 
-                const diffReview = await invokeDiffReviewLlm(prompt);
+                const diffReview = await invokeDiffReviewLlm(reviewAgentLogPath, prompt);
                 reviews.push(diffReview);
             } catch (error) {
                 console.error(`Error fetching file content for ${file_diff.to}: ${error}`);

packages/web/src/features/agents/review-agent/nodes/githubPrParser.ts

@@ -1,18 +1,14 @@
 import { sourcebot_pr_payload, sourcebot_file_diff, sourcebot_diff } from "@/features/agents/review-agent/types";
-import { WebhookEventDefinition } from "@octokit/webhooks/types";
 import parse from "parse-diff";
 import { Octokit } from "octokit";
+import { GitHubPullRequest } from "@/features/agents/review-agent/types";
 
-export const githubPrParser = async (octokit: Octokit, payload: WebhookEventDefinition<"pull-request-opened"> | WebhookEventDefinition<"pull-request-synchronize">): Promise<sourcebot_pr_payload> => {
+export const githubPrParser = async (octokit: Octokit, pullRequest: GitHubPullRequest): Promise<sourcebot_pr_payload> => {
     console.log("Executing github_pr_parser");
 
-    if (!payload.installation) {
-        throw new Error("Installation not found in github payload");
-    }
-
     let parsedDiff: parse.File[] = [];  
     try {
-        const diff = await octokit.request(payload.pull_request.patch_url);
+        const diff = await octokit.request(pullRequest.diff_url);
         parsedDiff = parse(diff.data);
     } catch (error) {
         console.error("Error fetching diff: ", error);
@@ -56,14 +52,13 @@ export const githubPrParser = async (octokit: Octokit, payload: WebhookEventDefi
 
     console.log("Completed github_pr_parser");
     return {
-        title: payload.pull_request.title,
-        description: payload.pull_request.body ?? "",
+        title: pullRequest.title,
+        description: pullRequest.body ?? "",
         hostDomain: "github.com",
-        owner: payload.repository.owner.login,
-        repo: payload.repository.name,
+        owner: pullRequest.base.repo.owner.login,
+        repo: pullRequest.base.repo.name,
         file_diffs: filteredSourcebotFileDiffs,
-        number: payload.pull_request.number,
-        head_sha: payload.pull_request.head.sha,
-        installation_id: payload.installation!.id,
+        number: pullRequest.number,
+        head_sha: pullRequest.head.sha
     }
 }

packages/web/src/features/agents/review-agent/nodes/invokeDiffReviewLlm.ts

@@ -1,8 +1,9 @@
 import OpenAI from "openai";
 import { sourcebot_diff_review_schema, sourcebot_diff_review } from "@/features/agents/review-agent/types";
 import { env } from "@/env.mjs";
+import fs from "fs";
 
-export const invokeDiffReviewLlm = async (prompt: string): Promise<sourcebot_diff_review> => {
+export const invokeDiffReviewLlm = async (reviewAgentLogPath: string | undefined, prompt: string): Promise<sourcebot_diff_review> => {
     console.log("Executing invoke_diff_review_llm");
 
     if (!env.OPENAI_API_KEY) {
@@ -14,7 +15,9 @@ export const invokeDiffReviewLlm = async (prompt: string): Promise<sourcebot_dif
         apiKey: env.OPENAI_API_KEY,
     });
 
-    console.log("Prompt: ", prompt);
+    if (reviewAgentLogPath) {
+        fs.appendFileSync(reviewAgentLogPath, `\n\nPrompt:\n${prompt}`);
+    }
 
     try {
         const completion = await openai.chat.completions.create({
@@ -25,7 +28,9 @@ export const invokeDiffReviewLlm = async (prompt: string): Promise<sourcebot_dif
         });
 
         const openaiResponse = completion.choices[0].message.content;
-        console.log("OpenAI response: ", openaiResponse);
+        if (reviewAgentLogPath) {
+            fs.appendFileSync(reviewAgentLogPath, `\n\nResponse:\n${openaiResponse}`);
+        }
 
         const diffReviewJson = JSON.parse(openaiResponse || '{}');
         const diffReview = sourcebot_diff_review_schema.safeParse(diffReviewJson);