diff --git a/src/aggregator/checkly-aggregator.ts b/src/aggregator/checkly-aggregator.ts index 04d5fd6..f0d6ff2 100644 --- a/src/aggregator/checkly-aggregator.ts +++ b/src/aggregator/checkly-aggregator.ts @@ -1,47 +1,10 @@ import { CheckContext, ContextKey } from "./ContextAggregator"; import { checkly } from "../checkly/client"; import { WebhookAlertDto } from "../checkly/alertDTO"; -import { Check, CheckResult } from "../checkly/models"; - -const getCheckLogs = async (checkId: string, checkResultId: string) => { - const logs = await checkly.getCheckResult(checkId, checkResultId); - console.log("logs"); - console.log(logs); - - return logs; -}; - -const mapCheckToContextValue = (check: Check) => { - return { - checkId: check.id, - type: check.checkType, - frequency: check.frequency, - frequencyOffset: check.frequencyOffset, - shouldFail: check.shouldFail, - locations: check.locations, - tags: check.tags, - maxResponseTime: check.maxResponseTime, - sslCheckDomain: check.sslCheckDomain, - retryStrategy: check.retryStrategy, - }; -}; - -const mapCheckResultToContextValue = (result: CheckResult) => { - return { - resultId: result.id, - hasErrors: result.hasErrors, - hasFailures: result.hasFailures, - runLocation: result.runLocation, - startedAt: result.startedAt, - stoppedAt: result.stoppedAt, - responseTime: result.responseTime, - checkId: result.checkId, - attempts: result.attempts, - isDegraded: result.isDegraded, - overMaxResponseTime: result.overMaxResponseTime, - resultType: result.resultType, - }; -}; +import { + mapCheckResultToContextValue, + mapCheckToContextValue, +} from "../checkly/utils"; export const checklyAggregator = { fetchContext: async (alert: WebhookAlertDto): Promise => { diff --git a/src/ai/utils.ts b/src/ai/utils.ts index a405edd..e95e224 100644 --- a/src/ai/utils.ts +++ b/src/ai/utils.ts @@ -33,8 +33,6 @@ export const formatToolOutput = ( toolCallId: string, output: unknown ): RunSubmitToolOutputsParams.ToolOutput => { - console.log("output", output); - return { output: JSON.stringify(output), tool_call_id: toolCallId, diff --git a/src/checkly/utils.ts b/src/checkly/utils.ts new file mode 100644 index 0000000..83d39d3 --- /dev/null +++ b/src/checkly/utils.ts @@ -0,0 +1,33 @@ +import { Check, CheckResult } from "./models"; + +export const mapCheckToContextValue = (check: Check) => { + return { + checkId: check.id, + type: check.checkType, + frequency: check.frequency, + frequencyOffset: check.frequencyOffset, + shouldFail: check.shouldFail, + locations: check.locations, + tags: check.tags, + maxResponseTime: check.maxResponseTime, + sslCheckDomain: check.sslCheckDomain, + retryStrategy: check.retryStrategy, + }; +}; + +export const mapCheckResultToContextValue = (result: CheckResult) => { + return { + resultId: result.id, + hasErrors: result.hasErrors, + hasFailures: result.hasFailures, + runLocation: result.runLocation, + startedAt: result.startedAt, + stoppedAt: result.stoppedAt, + responseTime: result.responseTime, + checkId: result.checkId, + attempts: result.attempts, + isDegraded: result.isDegraded, + overMaxResponseTime: result.overMaxResponseTime, + resultType: result.resultType, + }; +}; diff --git a/src/routes/checklywebhook.ts b/src/routes/checklywebhook.ts index 1472015..18ebaeb 100644 --- a/src/routes/checklywebhook.ts +++ b/src/routes/checklywebhook.ts @@ -84,7 +84,7 @@ router.post("/", async (req: Request, res: Response) => { }); await app.client.chat.postMessage({ - channel: "C07V9GNU9L6", + channel: process.env.SLACK_ALERT_CHANNEL_ID as string, metadata: { event_type: "alert", event_payload: { diff --git a/src/slackbot/app.ts b/src/slackbot/app.ts index e192d3f..baaa996 100644 --- a/src/slackbot/app.ts +++ b/src/slackbot/app.ts @@ -1,45 +1,26 @@ -import { App, LogLevel } from "@slack/bolt"; +import { App } from "@slack/bolt"; import { getOpenaiClient, getOpenaiSDKClient } from "../ai/openai"; import { getRunMessages } from "../ai/utils"; import { SreAssistant } from "../sre-assistant/SreAssistant"; +import { getSlackConfig, validateConfig } from "./config"; +import { getThreadMetadata } from "./utils"; import GitHubAPI from "../github/github"; import { GithubAgent } from "../github/agent"; import moment from "moment"; -import { createReleaseBlock, divider as releaseDivider, releaseHeader } from "../github/slackBlock"; - -export const app = new App({ - signingSecret: process.env.SLACK_SIGNING_SECRET, - token: process.env.SLACK_AUTH_TOKEN, - appToken: process.env.SLACK_APP_TOKEN, - socketMode: true, - logLevel: - process.env.NODE_ENV !== "production" ? LogLevel.DEBUG : LogLevel.INFO, -}); - -app.command("/help123", async ({ command, ack }) => { - await ack(); - await app.client.chat.postEphemeral({ - channel: command.channel_id, - text: "hey", - user: command.user_id, - }); -}); - -app.message(`hey help`, async ({ message, context }) => { - await app.client.chat.postEphemeral({ - channel: message.channel, - text: "e", - user: context.userId!, - }); -}); - -app.message("Hey SREBot", async ({ say }) => { - await say("helloworld"); -}); +import { + createReleaseBlock, + divider as releaseDivider, + releaseHeader, +} from "../github/slackBlock"; + +// Initialize Slack app with validated configuration +const initializeSlackApp = () => { + const config = getSlackConfig(); + validateConfig(config); + return new App(config); +}; -app.message("whatismyuserid", async ({ context, say }) => { - await say(context.userId!); -}); +export const app = initializeSlackApp(); let setupAgent = () => { const CHECKLY_GITHUB_TOKEN = process.env.CHECKLY_GITHUB_TOKEN!; @@ -53,70 +34,81 @@ let setupAgent = () => { const githubAgent = setupAgent(); app.command("/srebot-releases", async ({ command, ack, respond }) => { - await ack(); - let summaries = await githubAgent.summarizeReleases(command.text, 'checkly'); - if (summaries.releases.length === 0) { - await respond({ text: `No releases found in repo ${summaries.repo} since ${summaries.since}`}); - } - - let releases = summaries.releases.sort((a, b) => new Date(b.release_date).getTime() - new Date(a.release_date).getTime()); let response = [releaseHeader].concat(releases.map(summary => { - const date = moment(summary.release_date).fromNow(); - const authors = summary.authors.filter(author => author !== null).map(author => author.login) - return createReleaseBlock({ - release: summary.id, - releaseUrl: summary.link, - diffUrl: summary.diffLink, - date, - repo: summaries.repo.name, - repoUrl: summaries.repo.link, - authors, - summary: summary.summary - }).blocks as any; - }).reduce((prev, curr) => { - if (!prev) { - return curr; - } - - return prev.concat([releaseDivider]).concat(curr); - })); - - await respond({ - blocks: response - }); -}) + await ack(); + let summaries = await githubAgent.summarizeReleases(command.text, "checkly"); + if (summaries.releases.length === 0) { + await respond({ + text: `No releases found in repo ${summaries.repo} since ${summaries.since}`, + }); + } + + let releases = summaries.releases.sort( + (a, b) => + new Date(b.release_date).getTime() - new Date(a.release_date).getTime() + ); + let response = [releaseHeader].concat( + releases + .map((summary) => { + const date = moment(summary.release_date).fromNow(); + const authors = summary.authors + .filter((author) => author !== null) + .map((author) => author.login); + return createReleaseBlock({ + release: summary.id, + releaseUrl: summary.link, + diffUrl: summary.diffLink, + date, + repo: summaries.repo.name, + repoUrl: summaries.repo.link, + authors, + summary: summary.summary, + }).blocks as any; + }) + .reduce((prev, curr) => { + if (!prev) { + return curr; + } + + return prev.concat([releaseDivider]).concat(curr); + }) + ); + + await respond({ + blocks: response, + }); +}); app.event("app_mention", async ({ event, context }) => { try { - let threadId; - let alertId = "test"; + let threadId, alertId; + const threadTs = (event as any).thread_ts || event.ts; + // Handle threaded conversations if ((event as any).thread_ts) { try { const result = await app.client.conversations.replies({ channel: event.channel, ts: (event as any).thread_ts, - limit: 1, include_all_metadata: true, }); - if (result.messages && result.messages.length > 0) { - const metadata = result.messages[0].metadata?.event_payload as { - threadId: string; - alertId: string; - }; - - threadId = metadata?.threadId; - alertId = metadata?.alertId; - } + + const { threadId: existingThreadId, alertId: existingAlertId } = + await getThreadMetadata(result.messages || []); + + threadId = existingThreadId; + alertId = existingAlertId; } catch (error) { - console.error("Error fetching parent message:", error); + console.error("Error fetching thread replies:", error); } } + // Create new thread if needed if (!threadId) { const thread = await getOpenaiClient().beta.threads.create(); threadId = thread.id; } + // Initialize assistant and process message const assistant = new SreAssistant(threadId, alertId, { username: event.user_profile?.display_name || @@ -125,28 +117,42 @@ app.event("app_mention", async ({ event, context }) => { "Unknown User", date: new Date().toISOString(), }); - const userMessage = await assistant.addMessage(event.text); - const responseMessages = await assistant - .runSync() - .then((run) => getRunMessages(threadId, run.id)); - const send = async (msg: string) => { + await assistant.addMessage(event.text); + const run = await assistant.runSync(); + const responseMessages = await getRunMessages(threadId, run.id); + + const sendMessage = (msg: string) => app.client.chat.postMessage({ token: context.botToken, channel: event.channel, text: msg, - thread_ts: (event as any).thread_ts || event.ts, + thread_ts: threadTs, + ...(threadId && { + metadata: { + event_type: "alert", + event_payload: { threadId }, + }, + }), }); - }; - await responseMessages.map((msg) => - send( - msg.content - .map((c) => (c.type === "text" ? c.text.value : "")) - .join("\n") + await Promise.all( + responseMessages.map((msg) => + sendMessage( + msg.content + .filter((c) => c.type === "text") + .map((c) => (c as any).text.value) + .join("") + ) ) ); } catch (error) { - console.error("Error reacting to mention:", error); + console.error("Error processing app mention:", error); + await app.client.chat.postMessage({ + token: context.botToken, + channel: event.channel, + text: "Sorry, I encountered an error while processing your request.", + thread_ts: (event as any).thread_ts || event.ts, + }); } }); diff --git a/src/slackbot/config.ts b/src/slackbot/config.ts new file mode 100644 index 0000000..d89bd1d --- /dev/null +++ b/src/slackbot/config.ts @@ -0,0 +1,36 @@ +import { LogLevel } from "@slack/bolt"; + +interface SlackConfig { + signingSecret: string; + token: string; + appToken: string; + socketMode: boolean; + logLevel: LogLevel; +} + +export const getSlackConfig = (): SlackConfig => ({ + signingSecret: process.env.SLACK_SIGNING_SECRET!, + token: process.env.SLACK_AUTH_TOKEN!, + appToken: process.env.SLACK_APP_TOKEN!, + socketMode: true, + logLevel: + process.env.NODE_ENV !== "production" ? LogLevel.DEBUG : LogLevel.INFO, +}); + +export const validateConfig = (config: SlackConfig): void => { + const requiredEnvVars = [ + "SLACK_SIGNING_SECRET", + "SLACK_AUTH_TOKEN", + "SLACK_APP_TOKEN", + ]; + + const missingVars = requiredEnvVars.filter( + (varName) => !process.env[varName] + ); + + if (missingVars.length > 0) { + throw new Error( + `Missing required environment variables: ${missingVars.join(", ")}` + ); + } +}; diff --git a/src/slackbot/utils.ts b/src/slackbot/utils.ts new file mode 100644 index 0000000..6fd03ac --- /dev/null +++ b/src/slackbot/utils.ts @@ -0,0 +1,17 @@ +export const getThreadMetadata = async (messages: any[]) => { + let threadId, alertId; + + if (messages && messages.length > 0) { + const firstBotMessage = messages.find((msg) => msg.bot_id); + if (firstBotMessage) { + const metadata = firstBotMessage.metadata?.event_payload as { + threadId: string; + alertId: string; + }; + threadId = metadata?.threadId; + alertId = metadata?.alertId; + } + } + + return { threadId, alertId }; +}; diff --git a/src/sre-assistant/SreAssistant.ts b/src/sre-assistant/SreAssistant.ts index 5a3d7d3..09a9ea6 100644 --- a/src/sre-assistant/SreAssistant.ts +++ b/src/sre-assistant/SreAssistant.ts @@ -3,9 +3,11 @@ import { Tool } from "../ai/Tool"; import type { RunCreateParams } from "openai/resources/beta/threads"; import { SearchContextTool } from "./tools/SearchContextTool"; import { GithubAgentInteractionTool } from "./tools/GithubAgentInteractionTool"; +import { ChecklyTool } from "./tools/ChecklyTool"; +import { prisma } from "../prisma"; export class SreAssistant extends BaseAssistant { - alertId: string; + alertId: string | undefined; interactionContext: { username: string; date: string; @@ -13,7 +15,7 @@ export class SreAssistant extends BaseAssistant { constructor( threadId: string, - alertId: string, + alertId: string | undefined = undefined, interactionContext: { username: string; date: string; @@ -30,6 +32,20 @@ export class SreAssistant extends BaseAssistant { } protected async getInstructions(): Promise { + let alertSummary = ""; + if (this.alertId) { + const alert = await prisma.alert.findUniqueOrThrow({ + where: { + id: this.alertId, + }, + select: { + summary: true, + }, + }); + + alertSummary = alert.summary; + } + return `You are an AI-powered SRE Bot designed to assist in real-time incident management. Your primary goal is to reduce Mean Time To Resolution (MTTR) by automatically aggregating and analyzing contextual data, providing actionable insights, and guiding first responders effectively. Important reminders: @@ -38,18 +54,29 @@ Important reminders: - If you're unsure about any aspect, clearly state your level of confidence - Maintain a professional and calm tone throughout your responses - Focus on providing actionable information that can help reduce MTTR +- Load the check to see the script and understand the context and why the check is failing +- The user is a experienced devops engineer. Don't overcomplicate it, focus on the context and provide actionable insights. They know what they are doing, don't worry about the details. Interaction Context: Username: ${this.interactionContext["Username"]} Date: ${this.interactionContext["Date"]} +${alertSummary.length > 0 ? `Alert Summary:\n${alertSummary}` : ""} + Format your responses as slack mrkdwn messages and keep the answer concise and relevant.`; } protected async getTools(): Promise { + if (!this.alertId) { + return [new ChecklyTool(this), new GithubAgentInteractionTool(this)]; + } + const searchContextTool = new SearchContextTool(this); await searchContextTool.init(); - - return [new SearchContextTool(this), new GithubAgentInteractionTool(this)]; + return [ + searchContextTool, + new GithubAgentInteractionTool(this), + new ChecklyTool(this), + ]; } } diff --git a/src/sre-assistant/tools/ChecklyTool.ts b/src/sre-assistant/tools/ChecklyTool.ts new file mode 100644 index 0000000..c112f9d --- /dev/null +++ b/src/sre-assistant/tools/ChecklyTool.ts @@ -0,0 +1,113 @@ +import { z } from "zod"; +import { Tool, createToolParameters, createToolOutput } from "../../ai/Tool"; +import { SreAssistant } from "../SreAssistant"; +import { checkly } from "../../checkly/client"; +import { stringify } from "yaml"; +import { + mapCheckResultToContextValue, + mapCheckToContextValue, +} from "../../checkly/utils"; +import { generateObject } from "ai"; +import { getOpenaiSDKClient } from "../../ai/openai"; + +const parameters = createToolParameters( + z.object({ + action: z + .enum([ + "getCheck", + "getCheckResult", + "getAllFailingChecks", + "searchCheck", + ]) + .describe("The action to perform on the Checkly API"), + checkId: z + .string() + .describe( + "The ID of the Check to get information about. Omit this field for the 'getChecksStatus' action." + ) + .optional(), + query: z + .string() + .describe( + "A query to search for checks. Use this field only for the 'searchCheck' action." + ) + .optional(), + }) +); + +const outputSchema = createToolOutput( + z.string().describe("The response from the Checkly API") +); + +export class ChecklyTool extends Tool< + typeof parameters, + typeof outputSchema, + SreAssistant +> { + static parameters = parameters; + static outputSchema = outputSchema; + + constructor(agent: SreAssistant) { + super({ + name: "ChecklyAPI", + description: + "Interact with the Checkly API to retrieve relevant context about checks and check results.", + parameters, + agent, + }); + } + + async execute(input: z.infer) { + if (input.action === "getCheck") { + const check = await checkly.getCheck(input.checkId!); + return stringify({ + ...mapCheckToContextValue(check), + script: check.script, + }); + } else if (input.action === "getCheckResult") { + const results = await checkly + .getCheckResults(input.checkId!, undefined, 1) + .then((result) => { + return result[0]; + }); + + if (!results) { + return "No results found"; + } + + return stringify(mapCheckResultToContextValue(results)); + } else if (input.action === "getAllFailingChecks") { + const status = await checkly.getPrometheusCheckStatus(); + return stringify(status.failing); + } else if (input.action === "searchCheck") { + const checks = await checkly.getChecks(); + const search = await generateObject({ + model: getOpenaiSDKClient()("gpt-4o"), + prompt: `You are the Checkly Check Search Engine. You are given a query and a list of checks. Return the most relevant check that relates to the query. + + Available checks: ${stringify( + checks.map((c) => ({ ...mapCheckToContextValue(c) })) + )} + + Search Query: ${input.query ?? ""}`, + schema: z.object({ + checkName: z.string(), + checkId: z.string(), + }), + }); + + const relevantCheck = checks.find((c) => c.id === search.object.checkId); + + if (!relevantCheck) { + return "No relevant check found"; + } + + return stringify({ + ...mapCheckToContextValue(relevantCheck), + script: relevantCheck.script, + }); + } + + return "Invalid action"; + } +}