Skip to content

Commit

Permalink
Fixed alerting health check behavior when alerting cannot find its he…
Browse files Browse the repository at this point in the history 
…alth task in Task Manager. (#99564)

* Fixed alerting health check behavior when alerting cannot find its health task in Task Manager.

* fixed test

* added unit tests
  • Loading branch information
@YulNaumenko
YulNaumenko committed May 10, 2021
1 parent a574539 commit 36e7b27
Show file tree
Hide file tree
Showing 6 changed files with 290 additions and 48 deletions.
74 changes: 72 additions & 2 deletions x-pack/plugins/alerting/server/health/get_health.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,12 @@
* 2.0.
*/

import { savedObjectsRepositoryMock } from '../../../../../src/core/server/mocks';
import {
savedObjectsRepositoryMock,
savedObjectsServiceMock,
} from '../../../../../src/core/server/mocks';
import { AlertExecutionStatusErrorReasons, HealthStatus } from '../types';
import { getHealth } from './get_health';
import { getAlertingHealthStatus, getHealth } from './get_health';

const savedObjectsRepository = savedObjectsRepositoryMock.create();

Expand Down Expand Up @@ -221,3 +224,70 @@ describe('getHealth()', () => {
});
});
});

describe('getAlertingHealthStatus()', () => {
test('return the proper framework state if some of alerts has a decryption error', async () => {
const savedObjects = savedObjectsServiceMock.createStartContract();
const lastExecutionDateError = new Date().toISOString();
savedObjectsRepository.find.mockResolvedValueOnce({
total: 1,
per_page: 1,
page: 1,
saved_objects: [
{
id: '1',
type: 'alert',
attributes: {
alertTypeId: 'myType',
schedule: { interval: '10s' },
params: {
bar: true,
},
createdAt: new Date().toISOString(),
actions: [
{
group: 'default',
actionRef: 'action_0',
params: {
foo: true,
},
},
],
executionStatus: {
status: 'error',
lastExecutionDate: lastExecutionDateError,
error: {
reason: AlertExecutionStatusErrorReasons.Decrypt,
message: 'Failed decrypt',
},
},
},
score: 1,
references: [
{
name: 'action_0',
type: 'action',
id: '1',
},
],
},
],
});
savedObjectsRepository.find.mockResolvedValue({
total: 0,
per_page: 10,
page: 1,
saved_objects: [],
});
const result = await getAlertingHealthStatus(
{ ...savedObjects, createInternalRepository: () => savedObjectsRepository },
1
);
expect(result).toStrictEqual({
state: {
runs: 2,
health_status: HealthStatus.Warning,
},
});
});
});
15 changes: 14 additions & 1 deletion x-pack/plugins/alerting/server/health/get_health.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* 2.0.
*/

import { ISavedObjectsRepository } from 'src/core/server';
import { ISavedObjectsRepository, SavedObjectsServiceStart } from 'src/core/server';
import { AlertsHealth, HealthStatus, RawAlert, AlertExecutionStatusErrorReasons } from '../types';

export const getHealth = async (
Expand Down Expand Up @@ -97,3 +97,16 @@ export const getHealth = async (

return healthStatuses;
};

export const getAlertingHealthStatus = async (
savedObjects: SavedObjectsServiceStart,
stateRuns?: number
) => {
const alertingHealthStatus = await getHealth(savedObjects.createInternalRepository(['alert']));
return {
state: {
runs: (stateRuns || 0) + 1,
health_status: alertingHealthStatus.decryptionHealth.status,
},
};
};
166 changes: 148 additions & 18 deletions x-pack/plugins/alerting/server/health/get_state.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,16 @@ import {
} from './get_state';
import { ConcreteTaskInstance, TaskStatus } from '../../../task_manager/server';
import { HealthStatus } from '../types';
import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks';

jest.mock('./get_health', () => ({
getAlertingHealthStatus: jest.fn().mockReturnValue({
state: {
runs: 0,
health_status: 'warn',
},
}),
}));

const tick = () => new Promise((resolve) => setImmediate(resolve));

Expand All @@ -38,6 +48,9 @@ const getHealthCheckTask = (overrides = {}): ConcreteTaskInstance => ({
...overrides,
});

const logger = loggingSystemMock.create().get();
const savedObjects = savedObjectsServiceMock.createStartContract();

describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
beforeEach(() => jest.useFakeTimers());

Expand All @@ -47,7 +60,21 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
const pollInterval = 100;
const halfInterval = Math.floor(pollInterval / 2);

getHealthStatusStream(mockTaskManager, pollInterval).subscribe();
getHealthStatusStream(
mockTaskManager,
logger,
savedObjects,
Promise.resolve({
healthCheck: {
interval: '5m',
},
invalidateApiKeysTask: {
interval: '5m',
removalDelay: '1h',
},
}),
pollInterval
).subscribe();

// shouldn't fire before poll interval passes
// should fire once each poll interval
Expand All @@ -68,7 +95,22 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
const pollInterval = 100;
const halfInterval = Math.floor(pollInterval / 2);

getHealthStatusStream(mockTaskManager, pollInterval, retryDelay).subscribe();
getHealthStatusStream(
mockTaskManager,
logger,
savedObjects,
Promise.resolve({
healthCheck: {
interval: '5m',
},
invalidateApiKeysTask: {
interval: '5m',
removalDelay: '1h',
},
}),
pollInterval,
retryDelay
).subscribe();

jest.advanceTimersByTime(halfInterval);
expect(mockTaskManager.get).toHaveBeenCalledTimes(0);
Expand Down Expand Up @@ -99,7 +141,18 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
mockTaskManager.get.mockResolvedValue(getHealthCheckTask());

const status = await getHealthServiceStatusWithRetryAndErrorHandling(
mockTaskManager
mockTaskManager,
logger,
savedObjects,
Promise.resolve({
healthCheck: {
interval: '5m',
},
invalidateApiKeysTask: {
interval: '5m',
removalDelay: '1h',
},
})
).toPromise();

expect(status.level).toEqual(ServiceStatusLevels.available);
Expand All @@ -118,7 +171,18 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
);

const status = await getHealthServiceStatusWithRetryAndErrorHandling(
mockTaskManager
mockTaskManager,
logger,
savedObjects,
Promise.resolve({
healthCheck: {
interval: '5m',
},
invalidateApiKeysTask: {
interval: '5m',
removalDelay: '1h',
},
})
).toPromise();

expect(status.level).toEqual(ServiceStatusLevels.degraded);
Expand All @@ -137,7 +201,18 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
);

const status = await getHealthServiceStatusWithRetryAndErrorHandling(
mockTaskManager
mockTaskManager,
logger,
savedObjects,
Promise.resolve({
healthCheck: {
interval: '5m',
},
invalidateApiKeysTask: {
interval: '5m',
removalDelay: '1h',
},
})
).toPromise();

expect(status.level).toEqual(ServiceStatusLevels.unavailable);
Expand All @@ -152,12 +227,25 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
.mockRejectedValueOnce(new Error('Failure'))
.mockResolvedValue(getHealthCheckTask());

getHealthServiceStatusWithRetryAndErrorHandling(mockTaskManager, retryDelay).subscribe(
(status) => {
expect(status.level).toEqual(ServiceStatusLevels.available);
expect(status.summary).toEqual('Alerting framework is available');
}
);
getHealthServiceStatusWithRetryAndErrorHandling(
mockTaskManager,
logger,
savedObjects,
Promise.resolve({
healthCheck: {
interval: '5m',
},
invalidateApiKeysTask: {
interval: '5m',
removalDelay: '1h',
},
}),
retryDelay
).subscribe((status) => {
expect(status.level).toEqual(ServiceStatusLevels.available);
expect(logger.warn).toHaveBeenCalledTimes(1);
expect(status.summary).toEqual('Alerting framework is available');
});

await tick();
jest.advanceTimersByTime(retryDelay * 2);
Expand All @@ -169,18 +257,60 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => {
const mockTaskManager = taskManagerMock.createStart();
mockTaskManager.get.mockRejectedValue(err);

getHealthServiceStatusWithRetryAndErrorHandling(mockTaskManager, retryDelay).subscribe(
(status) => {
expect(status.level).toEqual(ServiceStatusLevels.unavailable);
expect(status.summary).toEqual('Alerting framework is unavailable');
expect(status.meta).toEqual({ error: err });
}
);
getHealthServiceStatusWithRetryAndErrorHandling(
mockTaskManager,
logger,
savedObjects,
Promise.resolve({
healthCheck: {
interval: '5m',
},
invalidateApiKeysTask: {
interval: '5m',
removalDelay: '1h',
},
}),
retryDelay
).subscribe((status) => {
expect(status.level).toEqual(ServiceStatusLevels.unavailable);
expect(status.summary).toEqual('Alerting framework is unavailable');
expect(status.meta).toEqual({ error: err });
});

for (let i = 0; i < MAX_RETRY_ATTEMPTS + 1; i++) {
await tick();
jest.advanceTimersByTime(retryDelay);
}
expect(mockTaskManager.get).toHaveBeenCalledTimes(MAX_RETRY_ATTEMPTS + 1);
});

it('should schedule a new health check task if it does not exist without throwing an error', async () => {
const mockTaskManager = taskManagerMock.createStart();
mockTaskManager.get.mockRejectedValue({
output: {
statusCode: 404,
message: 'Not Found',
},
});

const status = await getHealthServiceStatusWithRetryAndErrorHandling(
mockTaskManager,
logger,
savedObjects,
Promise.resolve({
healthCheck: {
interval: '5m',
},
invalidateApiKeysTask: {
interval: '5m',
removalDelay: '1h',
},
})
).toPromise();

expect(mockTaskManager.ensureScheduled).toHaveBeenCalledTimes(1);
expect(status.level).toEqual(ServiceStatusLevels.degraded);
expect(status.summary).toEqual('Alerting framework is degraded');
expect(status.meta).toBeUndefined();
});
});
Loading

0 comments on commit 36e7b27

Please sign in to comment.