Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FEATURE core-incidents] Poll for active incidents #73

Merged
merged 10 commits into from
Jan 6, 2019
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ PAGERBEAUTY_PD_SCHEDULES=SCHEDL1,SCHEDL2
# Default: 10.
# PAGERBEAUTY_REFRESH_RATE_MINUTES=10

# (Optional) Disable polling for active incidents.
# Default: false
# PAGERBEAUTY_INCIDENTS_DISABLE=true

# (Optional) How often to refresh active incidents, in minutes.
# Default: 1
# PAGERBEAUTY_INCIDENTS_REFRESH_RATE_MINUTES=5

# (Optional) Highest logging level to include into application logs.
# One of: error, warn, info, verbose, debug, silly
# Default: info
Expand Down
3 changes: 2 additions & 1 deletion docker-compose.circleci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ services:
PAGERBEAUTY_PD_API_KEY: v2_api_key
# Mock Mock PD APInow only responds with these two schedules
PAGERBEAUTY_PD_SCHEDULES: P538IZH,PJ1P5JQ,P2RFGIP
# Faster refresh for dev server
# Faster refreshes for dev server
PAGERBEAUTY_REFRESH_RATE_MINUTES: 0.1
PAGERBEAUTY_INCIDENTS_REFRESH_RATE_MINUTES: 0.05
# Verbose
PAGERBEAUTY_LOG_LEVEL: silly
ports:
Expand Down
1 change: 1 addition & 0 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ services:
PAGERBEAUTY_PD_SCHEDULES: ${PAGERBEAUTY_PD_SCHEDULES:-P538IZH,PJ1P5JQ,P2RFGIP}
# Faster refresh for dev server
PAGERBEAUTY_REFRESH_RATE_MINUTES: ${PAGERBEAUTY_REFRESH_RATE_MINUTES:-0.1}
PAGERBEAUTY_INCIDENTS_REFRESH_RATE_MINUTES: ${PAGERBEAUTY_INCIDENTS_REFRESH_RATE_MINUTES:-0.05}
ports:
- "8080:8080"
env_file:
Expand Down
62 changes: 50 additions & 12 deletions src/app/PagerBeautyWorker.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ import logger from 'winston';

import { Timer } from './Timer';
import { PagerBeautyInitError } from '../errors';
import { IncidentsTimerTask } from '../tasks/IncidentsTimerTask';
import { OnCallsTimerTask } from '../tasks/OnCallsTimerTask';
import { SchedulesTimerTask } from '../tasks/SchedulesTimerTask';
import { IncidentsService } from '../services/IncidentsService';
import { OnCallsService } from '../services/OnCallsService';
import { SchedulesService } from '../services/SchedulesService';
import { PagerDutyClient } from '../services/PagerDutyClient';
Expand Down Expand Up @@ -37,26 +39,37 @@ export class PagerBeautyWorker {
);
this.onCallsService = new OnCallsService(this.pagerDutyClient);
this.schedulesService = new SchedulesService(this.pagerDutyClient);
// Optional
this.incidentsService = false;

// Timers
this.onCallsTimer = false;
this.schedulesTimer = false;

// Parse refresh interval
const refreshRateMinutes = Number(pagerDutyConfig.schedules.refreshRate);
if (Number.isNaN(refreshRateMinutes)) {
throw new PagerBeautyInitError('Refresh rate is not a number');
}
this.refreshRateMS = refreshRateMinutes * 60 * 1000;
this.incidentsTimer = false;

// Requested schedules list.
this.schedulesList = pagerDutyConfig.schedules.list;

// Schedules and on-calls poll interval.
this.schedulesRefreshMS = PagerBeautyWorker.refreshRateToMs(
pagerDutyConfig.schedules.refreshRate,
);

// Incidents are optional
this.incidentsEnabled = pagerDutyConfig.incidents.enabled;
// Incidents poll interval.
if (this.incidentsEnabled) {
this.incidentsRefreshMS = PagerBeautyWorker.refreshRateToMs(
pagerDutyConfig.incidents.refreshRate,
);
this.incidentsService = new IncidentsService(this.pagerDutyClient);
}
}

// ------- Public API -------------------------------------------------------

async start() {
const { db } = this;
const { db, incidentsEnabled } = this;
logger.debug('Initializing database.');
db.set('oncalls', new Map());
db.set('schedules', new Map());
Expand All @@ -66,6 +79,12 @@ export class PagerBeautyWorker {

// Then load on-calls.
await this.startOnCallsWorker();

// Incidents
if (incidentsEnabled) {
// No need to await on incidents.
this.startIncidentsWorker();
}
return true;
}

Expand All @@ -85,26 +104,45 @@ export class PagerBeautyWorker {
// ------- Internal machinery -----------------------------------------------

async startSchedulesWorker() {
const { refreshRateMS, schedulesList } = this;
const { schedulesRefreshMS, schedulesList } = this;
const schedulesTimerTask = new SchedulesTimerTask({
db: this.db,
schedulesService: this.schedulesService,
schedulesList,
});
this.schedulesTimer = new Timer(schedulesTimerTask, refreshRateMS);
this.schedulesTimer = new Timer(schedulesTimerTask, schedulesRefreshMS);
await this.schedulesTimer.start();
}

async startOnCallsWorker() {
const { refreshRateMS } = this;
const { schedulesRefreshMS } = this;
const onCallsTimerTask = new OnCallsTimerTask({
db: this.db,
onCallsService: this.onCallsService,
});
this.onCallsTimer = new Timer(onCallsTimerTask, refreshRateMS);
this.onCallsTimer = new Timer(onCallsTimerTask, schedulesRefreshMS);
await this.onCallsTimer.start();
}

async startIncidentsWorker() {
const { incidentsRefreshMS } = this;
const incidentsTimerTask = new IncidentsTimerTask({
db: this.db,
incidentsService: this.incidentsService,
});
this.incidentsTimer = new Timer(incidentsTimerTask, incidentsRefreshMS);
await this.incidentsTimer.start();
}

static refreshRateToMs(minutesStr) {
// String minutes to integer milliseconds.
const minutes = Number(minutesStr);
if (Number.isNaN(minutes)) {
throw new PagerBeautyInitError(`Incorrect refresh rate: ${minutesStr}`);
}
return minutes * 60 * 1000;
}

// ------- Class end --------------------------------------------------------
}

Expand Down
58 changes: 58 additions & 0 deletions src/models/Incident.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// ------- Imports -------------------------------------------------------------

// This model to be compatible both with backend and frontend.

// ------- OnCall --------------------------------------------------------------

export class Incident {
constructor({
id,
status,
scheduleId,
title,
summary,
url,
serviceName,
}) {
this.id = id;
this.scheduleId = scheduleId;
this.status = status;
this.title = title;
this.summary = summary;
this.url = url;
this.serviceName = serviceName;
}

serialize() {
return {
id: this.id,
scheduleId: this.scheduleId,
status: this.status,
title: this.title,
summary: this.summary,
serviceName: this.serviceName,
url: this.url,
};
}

toString() {
return JSON.stringify(this.serialize());
}

static fromApiRecord(record, scheduleId) {
const attributes = {
id: record.id,
status: record.status,
description: record.description,
title: record.title,
summary: record.summary,
url: record.html_url,
serviceName: record.service ? record.service.summary : 'Unknown',
scheduleId,
};
return new Incident(attributes);
}
// ------- Class end --------------------------------------------------------
}

// ------- End -----------------------------------------------------------------
10 changes: 10 additions & 0 deletions src/models/OnCall.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ export class OnCall {
} else {
this.schedule = new Schedule(schedule);
}
this.incident = false;
}

serialize() {
Expand All @@ -45,13 +46,22 @@ export class OnCall {
dateStart: this.dateStart.utc(),
dateEnd: this.dateEnd.utc(),
schedule: this.schedule.serialize(),
incident: this.incident ? this.incident.serialize() : null,
};
}

toString() {
return JSON.stringify(this.serialize());
}

setIncident(incident) {
this.incident = incident;
}

clearIncident() {
this.incident = false;
}

userAvatarSized(size = 2048) {
const url = new URL(this.userAvatarURL);
const searchParams = new URLSearchParams(url.searchParams);
Expand Down
7 changes: 7 additions & 0 deletions src/models/Schedule.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,15 @@ export class Schedule {
url,
summary,
description,
escalationPolicies = [],
}) {
this.id = id;
this.name = name;
this.url = url;
this.timezone = timezone;
this.summary = summary;
this.description = description;
this.escalationPolicies = new Set(escalationPolicies);
}

serialize() {
Expand All @@ -29,6 +31,7 @@ export class Schedule {
timezone: this.timezone,
summary: this.summary,
description: this.description,
escalationPolicies: Array.from(this.escalationPolicies),
};
}

Expand All @@ -45,6 +48,10 @@ export class Schedule {
summary: record.summary,
description: record.description,
};
const escalationPolicies = record.escalation_policies;
if (escalationPolicies) {
attributes.escalationPolicies = escalationPolicies.map(policy => policy.id);
}
return new Schedule(attributes);
}
// ------- Class end --------------------------------------------------------
Expand Down
4 changes: 4 additions & 0 deletions src/pagerbeauty.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ const config = {
list: process.env.PAGERBEAUTY_PD_SCHEDULES.replace(/\s*/g, '').split(','),
refreshRate: process.env.PAGERBEAUTY_REFRESH_RATE_MINUTES || 10,
},
incidents: {
enabled: !process.env.PAGERBEAUTY_INCIDENTS_DISABLE,
refreshRate: process.env.PAGERBEAUTY_INCIDENTS_REFRESH_RATE_MINUTES || 1,
},
},
auth: {
name: process.env.PAGERBEAUTY_HTTP_USER,
Expand Down
74 changes: 74 additions & 0 deletions src/services/IncidentsService.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// ------- Imports -------------------------------------------------------------

import logger from 'winston';

// ------- Internal imports ----------------------------------------------------

import { Incident } from '../models/Incident';

// ------- IncidentsService ----------------------------------------------------

export class IncidentsService {
constructor(pagerDutyClient) {
this.client = pagerDutyClient;
this.incidentsRepo = new Map();
}

async load(onCallsService) {
const onCalls = onCallsService.onCallRepo;
if (!onCalls.size) {
logger.verbose('Skipping incidents load: OnCalls not loaded yet');
return false;
}

const missingSchedules = new Set();

for (const [scheduleId, onCall] of onCalls.entries()) {
try {
// Limit the number of requests by sending them in sync.
// eslint-disable-next-line no-await-in-loop
const record = await this.client.getActiveIncidentForUserOnSchedule(
onCall.userId,
onCall.schedule.escalationPolicies,
);

if (record) {
const incident = Incident.fromApiRecord(record, scheduleId);
this.incidentsRepo.set(scheduleId, incident);
// @todo: log when oncall already has an incident and it's different
onCall.setIncident(incident);
logger.verbose(
`Incident ${incident.id} is active for schedule ${scheduleId}`,
);
logger.silly(`Incident ${incident.toString()}`);
} else {
// No incidents, clear.
const existed = this.incidentsRepo.delete(scheduleId);
onCall.clearIncident();
if (existed) {
logger.verbose(`Incident resolved for schedule ${scheduleId}`);
}
}
} catch (e) {
logger.warn(
`Error loading incident for user ${onCall.userId} `
+ `on schedule ${scheduleId}: ${e}`,
);
}
}

if (missingSchedules.size) {
logger.warn(
`Missing incidents data for schedules: ${Array.from(missingSchedules).join()}`,
);
}
return true;
}

serialize() {
return Array.from(this.incidentsRepo.values(), r => r.serialize());
}
// ------- Class end --------------------------------------------------------
}

// ------- End -----------------------------------------------------------------
4 changes: 3 additions & 1 deletion src/services/OnCallsService.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@ export class OnCallsService {
}

if (missingSchedules.size) {
logger.warn(`Missing data for schedules: ${Array.from(missingSchedules).join()}`);
logger.warn(
`Missing oncall data for schedules: ${Array.from(missingSchedules).join()}`,
);
}
return true;
}
Expand Down
Loading