-
Notifications
You must be signed in to change notification settings - Fork 310
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Showing
7 changed files
with
178 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
86 changes: 8 additions & 78 deletions
86
yarn-project/end-to-end/src/quality_of_service/alert_checker.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,88 +1,18 @@ | ||
| import { type DebugLogger, createDebugLogger } from '@aztec/aztec.js'; | ||
| import { fileURLToPath } from '@aztec/foundation/url'; | ||
| import { createDebugLogger } from '@aztec/aztec.js'; | ||
|
|
||
| import * as fs from 'fs'; | ||
| import * as yaml from 'js-yaml'; | ||
| import { dirname, join } from 'path'; | ||
| import { AlertChecker } from './alert_checker.js'; | ||
|
|
||
| const GRAFANA_ENDPOINT = 'http://localhost:3000/api/datasources/proxy/uid/prometheus/api/v1/query'; | ||
| interface AlertConfig { | ||
| alert: string; | ||
| expr: string; | ||
| for: string; | ||
| labels: Record<string, string>; | ||
| annotations: Record<string, string>; | ||
| } | ||
| // Define __dirname for ES modules | ||
| const __filename = fileURLToPath(import.meta.url); | ||
| const __dirname = dirname(__filename); | ||
|
|
||
| // Load YAML configuration | ||
| function loadAlertsConfig(filePath: string): AlertConfig[] { | ||
| const fileContents = fs.readFileSync(join(__dirname, filePath), 'utf8'); | ||
| const data = yaml.load(fileContents) as { alerts: AlertConfig[] }; | ||
| return data.alerts; | ||
| } | ||
|
|
||
| // Function to query Grafana based on an expression | ||
| async function queryGrafana(expr: string): Promise<number> { | ||
| // Create base64 encoded credentials for basic auth | ||
| const credentials = Buffer.from('admin:admin').toString('base64'); | ||
| const ALERTS_FILE = process.env.ALERTS_FILE; | ||
|
|
||
| const response = await fetch(`${GRAFANA_ENDPOINT}?query=${encodeURIComponent(expr)}`, { | ||
| headers: { | ||
| Authorization: `Basic ${credentials}`, | ||
| }, | ||
| }); | ||
|
|
||
| if (!response.ok) { | ||
| throw new Error(`Failed to fetch data from Grafana: ${response.statusText}`); | ||
| } | ||
|
|
||
| const data = await response.json(); | ||
| const result = data.data.result; | ||
| return result.length > 0 ? parseFloat(result[0].value[1]) : 0; | ||
| if (!ALERTS_FILE) { | ||
| throw new Error('ALERTS_FILE is not set'); | ||
| } | ||
|
|
||
| // Function to check alerts based on expressions | ||
| async function checkAlerts(alerts: AlertConfig[], logger: DebugLogger) { | ||
| let alertTriggered = false; | ||
|
|
||
| for (const alert of alerts) { | ||
| logger.info(`Checking alert: ${JSON.stringify(alert)}`); | ||
|
|
||
| const metricValue = await queryGrafana(alert.expr); | ||
| logger.info(`Metric value: ${metricValue}`); | ||
| if (metricValue > 0) { | ||
| logger.error(`Alert ${alert.alert} triggered! Value: ${metricValue}`); | ||
| alertTriggered = true; | ||
| } else { | ||
| logger.info(`Alert ${alert.alert} passed.`); | ||
| } | ||
| } | ||
|
|
||
| // If any alerts have been triggered we fail the test | ||
| if (alertTriggered) { | ||
| throw new Error('Test failed due to triggered alert'); | ||
| } | ||
| } | ||
|
|
||
| // Main function to run tests | ||
| async function runAlertChecker(logger: DebugLogger) { | ||
| const alerts = loadAlertsConfig('alerts.yaml'); | ||
| try { | ||
| await checkAlerts(alerts, logger); | ||
| logger.info('All alerts passed.'); | ||
| } catch (error) { | ||
| logger.error(error instanceof Error ? error.message : String(error)); | ||
| process.exit(1); // Exit with error code | ||
| } | ||
| } | ||
|
|
||
| // Running as a jest test to use existing end to end test framework | ||
| describe('Alert Checker', () => { | ||
| const logger = createDebugLogger('aztec:alert-checker'); | ||
| const alertChecker = new AlertChecker(logger); | ||
|
|
||
| it('should check alerts', async () => { | ||
| await runAlertChecker(logger); | ||
| await alertChecker.runAlertCheckFromFilePath(ALERTS_FILE); | ||
| }); | ||
| }); |
96 changes: 96 additions & 0 deletions
96
yarn-project/end-to-end/src/quality_of_service/alert_checker.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| import { type DebugLogger } from '@aztec/aztec.js'; | ||
| import { fileURLToPath } from '@aztec/foundation/url'; | ||
|
|
||
| import * as fs from 'fs'; | ||
| import * as yaml from 'js-yaml'; | ||
| import { dirname, join } from 'path'; | ||
|
|
||
| export interface AlertConfig { | ||
| alert: string; | ||
| expr: string; | ||
| for: string; | ||
| labels: Record<string, string>; | ||
| annotations: Record<string, string>; | ||
| } | ||
|
|
||
| export interface AlertCheckerConfig { | ||
| grafanaEndpoint: string; | ||
| grafanaCredentials: string; | ||
| } | ||
|
|
||
| const DEFAULT_CONFIG: AlertCheckerConfig = { | ||
| grafanaEndpoint: 'http://localhost:3000/api/datasources/proxy/uid/prometheus/api/v1/query', | ||
| grafanaCredentials: 'admin:admin', | ||
| }; | ||
|
|
||
| export class AlertChecker { | ||
| private config: AlertCheckerConfig; | ||
| private logger: DebugLogger; | ||
|
|
||
| constructor(logger: DebugLogger, config: Partial<AlertCheckerConfig> = {}) { | ||
| this.config = { ...DEFAULT_CONFIG, ...config }; | ||
| this.logger = logger; | ||
| } | ||
|
|
||
| private loadAlertsConfig(filePath: string): AlertConfig[] { | ||
| const __filename = fileURLToPath(import.meta.url); | ||
| const __dirname = dirname(__filename); | ||
| const fileContents = fs.readFileSync(join(__dirname, filePath), 'utf8'); | ||
| const data = yaml.load(fileContents) as { alerts: AlertConfig[] }; | ||
| return data.alerts; | ||
| } | ||
|
|
||
| private async queryGrafana(expr: string): Promise<number> { | ||
| const credentials = Buffer.from(this.config.grafanaCredentials).toString('base64'); | ||
|
|
||
| const response = await fetch(`${this.config.grafanaEndpoint}?query=${encodeURIComponent(expr)}`, { | ||
| headers: { | ||
| Authorization: `Basic ${credentials}`, | ||
| }, | ||
| }); | ||
|
|
||
| if (!response.ok) { | ||
| throw new Error(`Failed to fetch data from Grafana: ${response.statusText}`); | ||
| } | ||
|
|
||
| const data = await response.json(); | ||
| const result = data.data.result; | ||
| return result.length > 0 ? parseFloat(result[0].value[1]) : 0; | ||
| } | ||
|
|
||
| private async checkAlerts(alerts: AlertConfig[]): Promise<void> { | ||
| let alertTriggered = false; | ||
|
|
||
| for (const alert of alerts) { | ||
| this.logger.info(`Checking alert: ${JSON.stringify(alert)}`); | ||
|
|
||
| const metricValue = await this.queryGrafana(alert.expr); | ||
| this.logger.info(`Metric value: ${metricValue}`); | ||
| if (metricValue > 0) { | ||
| this.logger.error(`Alert ${alert.alert} triggered! Value: ${metricValue}`); | ||
| alertTriggered = true; | ||
| } else { | ||
| this.logger.info(`Alert ${alert.alert} passed.`); | ||
| } | ||
| } | ||
|
|
||
| if (alertTriggered) { | ||
| throw new Error('Test failed due to triggered alert'); | ||
| } | ||
| } | ||
|
|
||
| public async runAlertCheck(alerts: AlertConfig[]): Promise<void> { | ||
| try { | ||
| await this.checkAlerts(alerts); | ||
| this.logger.info('All alerts passed.'); | ||
| } catch (error) { | ||
| this.logger.error(error instanceof Error ? error.message : String(error)); | ||
| throw error; | ||
| } | ||
| } | ||
|
|
||
| public async runAlertCheckFromFilePath(filePath: string): Promise<void> { | ||
| const alerts = this.loadAlertsConfig(filePath); | ||
| await this.checkAlerts(alerts); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters