diff --git a/x-pack/plugins/actions/kibana.json b/x-pack/plugins/actions/kibana.json index 0c40dec46a6ee..14ddb8257ff37 100644 --- a/x-pack/plugins/actions/kibana.json +++ b/x-pack/plugins/actions/kibana.json @@ -5,6 +5,6 @@ "kibanaVersion": "kibana", "configPath": ["xpack", "actions"], "requiredPlugins": ["licensing", "taskManager", "encryptedSavedObjects", "eventLog"], - "optionalPlugins": ["spaces"], + "optionalPlugins": ["usageCollection", "spaces"], "ui": false } diff --git a/x-pack/plugins/actions/server/plugin.test.ts b/x-pack/plugins/actions/server/plugin.test.ts index 91944dfa8f3ad..f55a5ca172144 100644 --- a/x-pack/plugins/actions/server/plugin.test.ts +++ b/x-pack/plugins/actions/server/plugin.test.ts @@ -11,8 +11,13 @@ import { licensingMock } from '../../licensing/server/mocks'; import { encryptedSavedObjectsMock } from '../../encrypted_saved_objects/server/mocks'; import { taskManagerMock } from '../../task_manager/server/mocks'; import { eventLogMock } from '../../event_log/server/mocks'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; describe('Actions Plugin', () => { + const usageCollectionMock: jest.Mocked = ({ + makeUsageCollector: jest.fn(), + registerCollector: jest.fn(), + } as unknown) as jest.Mocked; describe('setup()', () => { let context: PluginInitializerContext; let plugin: ActionsPlugin; @@ -23,11 +28,13 @@ describe('Actions Plugin', () => { context = coreMock.createPluginInitializerContext(); plugin = new ActionsPlugin(context); coreSetup = coreMock.createSetup(); + pluginsSetup = { taskManager: taskManagerMock.createSetup(), encryptedSavedObjects: encryptedSavedObjectsMock.createSetup(), licensing: licensingMock.createSetup(), eventLog: eventLogMock.createSetup(), + usageCollection: usageCollectionMock, }; }); @@ -108,6 +115,7 @@ describe('Actions Plugin', () => { encryptedSavedObjects: encryptedSavedObjectsMock.createSetup(), licensing: licensingMock.createSetup(), eventLog: eventLogMock.createSetup(), + usageCollection: usageCollectionMock, }; pluginsStart = { taskManager: taskManagerMock.createStart(), diff --git a/x-pack/plugins/actions/server/plugin.ts b/x-pack/plugins/actions/server/plugin.ts index b0555921f395f..5ea6320fbe54a 100644 --- a/x-pack/plugins/actions/server/plugin.ts +++ b/x-pack/plugins/actions/server/plugin.ts @@ -5,6 +5,7 @@ */ import { first, map } from 'rxjs/operators'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { PluginInitializerContext, Plugin, @@ -35,6 +36,7 @@ import { ActionTypeRegistry } from './action_type_registry'; import { ExecuteOptions } from './create_execute_function'; import { createExecuteFunction } from './create_execute_function'; import { registerBuiltInActionTypes } from './builtin_action_types'; +import { registerActionsUsageCollector } from './usage'; import { getActionsConfigurationUtilities } from './actions_config'; @@ -49,6 +51,7 @@ import { } from './routes'; import { LicenseState } from './lib/license_state'; import { IEventLogger, IEventLogService } from '../../event_log/server'; +import { initializeActionsTelemetry, scheduleActionsTelemetry } from './usage/task'; const EVENT_LOG_PROVIDER = 'actions'; export const EVENT_LOG_ACTIONS = { @@ -71,6 +74,7 @@ export interface ActionsPluginsSetup { licensing: LicensingPluginSetup; spaces?: SpacesPluginSetup; eventLog: IEventLogService; + usageCollection?: UsageCollectionSetup; } export interface ActionsPluginsStart { encryptedSavedObjects: EncryptedSavedObjectsPluginStart; @@ -91,6 +95,7 @@ export class ActionsPlugin implements Plugin, Plugi private spaces?: SpacesServiceSetup; private eventLogger?: IEventLogger; private isESOUsingEphemeralEncryptionKey?: boolean; + private readonly telemetryLogger: Logger; constructor(initContext: PluginInitializerContext) { this.config = initContext.config @@ -106,6 +111,7 @@ export class ActionsPlugin implements Plugin, Plugi .toPromise(); this.logger = initContext.logger.get('actions'); + this.telemetryLogger = initContext.logger.get('telemetry'); } public async setup(core: CoreSetup, plugins: ActionsPluginsSetup): Promise { @@ -140,6 +146,8 @@ export class ActionsPlugin implements Plugin, Plugi const actionExecutor = new ActionExecutor({ isESOUsingEphemeralEncryptionKey: this.isESOUsingEphemeralEncryptionKey, }); + + // get executions count const taskRunnerFactory = new TaskRunnerFactory(actionExecutor); const actionsConfigUtils = getActionsConfigurationUtilities( (await this.config) as ActionsConfig @@ -162,6 +170,20 @@ export class ActionsPlugin implements Plugin, Plugi actionsConfigUtils, }); + const usageCollection = plugins.usageCollection; + if (usageCollection) { + core.getStartServices().then(async ([coreStart, startPlugins]: [CoreStart, any]) => { + registerActionsUsageCollector(usageCollection, startPlugins.taskManager); + + initializeActionsTelemetry( + this.telemetryLogger, + plugins.taskManager, + core, + await this.kibanaIndex + ); + }); + } + core.http.registerRouteHandlerContext( 'actions', this.createRouteHandlerContext(await this.kibanaIndex) @@ -211,6 +233,8 @@ export class ActionsPlugin implements Plugin, Plugi getScopedSavedObjectsClient: core.savedObjects.getScopedClient, }); + scheduleActionsTelemetry(this.telemetryLogger, plugins.taskManager); + return { execute: createExecuteFunction({ taskManager: plugins.taskManager, diff --git a/x-pack/plugins/actions/server/usage/actions_telemetry.ts b/x-pack/plugins/actions/server/usage/actions_telemetry.ts new file mode 100644 index 0000000000000..ccdb4ecec2012 --- /dev/null +++ b/x-pack/plugins/actions/server/usage/actions_telemetry.ts @@ -0,0 +1,145 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { APICaller } from 'kibana/server'; + +export async function getTotalCount(callCluster: APICaller, kibanaIndex: string) { + const scriptedMetric = { + scripted_metric: { + init_script: 'state.types = [:]', + map_script: ` + String actionType = doc['action.actionTypeId'].value; + state.types.put(actionType, state.types.containsKey(actionType) ? state.types.get(actionType) + 1 : 1); + `, + // Combine script is executed per cluster, but we already have a key-value pair per cluster. + // Despite docs that say this is optional, this script can't be blank. + combine_script: 'return state', + // Reduce script is executed across all clusters, so we need to add up all the total from each cluster + // This also needs to account for having no data + reduce_script: ` + Map result = [:]; + for (Map m : states.toArray()) { + if (m !== null) { + for (String k : m.keySet()) { + result.put(k, result.containsKey(k) ? result.get(k) + m.get(k) : m.get(k)); + } + } + } + return result; + `, + }, + }; + + const searchResult = await callCluster('search', { + index: kibanaIndex, + rest_total_hits_as_int: true, + body: { + query: { + bool: { + filter: [{ term: { type: 'action' } }], + }, + }, + aggs: { + byActionTypeId: scriptedMetric, + }, + }, + }); + + return { + countTotal: Object.keys(searchResult.aggregations.byActionTypeId.value.types).reduce( + (total: number, key: string) => + parseInt(searchResult.aggregations.byActionTypeId.value.types[key], 0) + total, + 0 + ), + countByType: searchResult.aggregations.byActionTypeId.value.types, + }; +} + +export async function getInUseTotalCount(callCluster: APICaller, kibanaIndex: string) { + const scriptedMetric = { + scripted_metric: { + init_script: 'state.connectorIds = new HashMap(); state.total = 0;', + map_script: ` + String connectorId = doc['references.id'].value; + String actionRef = doc['references.name'].value; + if (state.connectorIds[connectorId] === null) { + state.connectorIds[connectorId] = actionRef; + state.total++; + } + `, + // Combine script is executed per cluster, but we already have a key-value pair per cluster. + // Despite docs that say this is optional, this script can't be blank. + combine_script: 'return state', + // Reduce script is executed across all clusters, so we need to add up all the total from each cluster + // This also needs to account for having no data + reduce_script: ` + Map connectorIds = [:]; + long total = 0; + for (state in states) { + if (state !== null) { + total += state.total; + for (String k : state.connectorIds.keySet()) { + connectorIds.put(k, connectorIds.containsKey(k) ? connectorIds.get(k) + state.connectorIds.get(k) : state.connectorIds.get(k)); + } + } + } + Map result = new HashMap(); + result.total = total; + result.connectorIds = connectorIds; + return result; + `, + }, + }; + + const actionResults = await callCluster('search', { + index: kibanaIndex, + rest_total_hits_as_int: true, + body: { + query: { + bool: { + filter: { + bool: { + must: { + nested: { + path: 'references', + query: { + bool: { + filter: { + bool: { + must: [ + { + term: { + 'references.type': 'action', + }, + }, + ], + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + aggs: { + refs: { + nested: { + path: 'references', + }, + aggs: { + actionRefIds: scriptedMetric, + }, + }, + }, + }, + }); + + return actionResults.aggregations.refs.actionRefIds.value.total; +} + +// TODO: Implement executions count telemetry with eventLog, when it will write to index diff --git a/x-pack/plugins/actions/server/usage/actions_usage_collector.test.ts b/x-pack/plugins/actions/server/usage/actions_usage_collector.test.ts new file mode 100644 index 0000000000000..214690383ceba --- /dev/null +++ b/x-pack/plugins/actions/server/usage/actions_usage_collector.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { registerActionsUsageCollector } from './actions_usage_collector'; +import { taskManagerMock } from '../../../task_manager/server/task_manager.mock'; + +const mockTaskManagerStart = taskManagerMock.start(); + +beforeEach(() => jest.resetAllMocks()); + +describe('registerActionsUsageCollector', () => { + let usageCollectionMock: jest.Mocked; + beforeEach(() => { + usageCollectionMock = ({ + makeUsageCollector: jest.fn(), + registerCollector: jest.fn(), + } as unknown) as jest.Mocked; + }); + + it('should call registerCollector', () => { + registerActionsUsageCollector(usageCollectionMock, mockTaskManagerStart); + expect(usageCollectionMock.registerCollector).toHaveBeenCalledTimes(1); + }); + + it('should call makeUsageCollector with type = actions', () => { + registerActionsUsageCollector(usageCollectionMock, mockTaskManagerStart); + expect(usageCollectionMock.makeUsageCollector).toHaveBeenCalledTimes(1); + expect(usageCollectionMock.makeUsageCollector.mock.calls[0][0].type).toBe('actions'); + }); +}); diff --git a/x-pack/plugins/actions/server/usage/actions_usage_collector.ts b/x-pack/plugins/actions/server/usage/actions_usage_collector.ts new file mode 100644 index 0000000000000..e298b3ad9d00c --- /dev/null +++ b/x-pack/plugins/actions/server/usage/actions_usage_collector.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { get } from 'lodash'; +import { TaskManagerStartContract } from '../../../task_manager/server'; +import { ActionsUsage } from './types'; + +export function createActionsUsageCollector( + usageCollection: UsageCollectionSetup, + taskManager: TaskManagerStartContract +) { + return usageCollection.makeUsageCollector({ + type: 'actions', + isReady: () => true, + fetch: async (): Promise => { + try { + const doc = await getLatestTaskState(await taskManager); + // get the accumulated state from the recurring task + const state: ActionsUsage = get(doc, 'state'); + + return { + ...state, + }; + } catch (err) { + return { + count_total: 0, + count_active_total: 0, + count_active_by_type: {}, + count_by_type: {}, + }; + } + }, + }); +} + +async function getLatestTaskState(taskManager: TaskManagerStartContract) { + try { + const result = await taskManager.get('Actions-actions_telemetry'); + return result; + } catch (err) { + const errMessage = err && err.message ? err.message : err.toString(); + /* + The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the + task manager has to wait for all plugins to initialize first. It's fine to ignore it as next time around it will be + initialized (or it will throw a different type of error) + */ + if (!errMessage.includes('NotInitialized')) { + throw err; + } + } + + return null; +} + +export function registerActionsUsageCollector( + usageCollection: UsageCollectionSetup, + taskManager: TaskManagerStartContract +) { + const collector = createActionsUsageCollector(usageCollection, taskManager); + usageCollection.registerCollector(collector); +} diff --git a/x-pack/plugins/actions/server/usage/index.ts b/x-pack/plugins/actions/server/usage/index.ts new file mode 100644 index 0000000000000..ddca1de3d6bda --- /dev/null +++ b/x-pack/plugins/actions/server/usage/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { registerActionsUsageCollector } from './actions_usage_collector'; diff --git a/x-pack/plugins/actions/server/usage/task.ts b/x-pack/plugins/actions/server/usage/task.ts new file mode 100644 index 0000000000000..a07a2aa8f1c70 --- /dev/null +++ b/x-pack/plugins/actions/server/usage/task.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger, CoreSetup } from 'kibana/server'; +import moment from 'moment'; +import { + RunContext, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../task_manager/server'; +import { getTotalCount, getInUseTotalCount } from './actions_telemetry'; + +export const TELEMETRY_TASK_TYPE = 'actions_telemetry'; + +export const TASK_ID = `Actions-${TELEMETRY_TASK_TYPE}`; + +export function initializeActionsTelemetry( + logger: Logger, + taskManager: TaskManagerSetupContract, + core: CoreSetup, + kibanaIndex: string +) { + registerActionsTelemetryTask(logger, taskManager, core, kibanaIndex); +} + +export function scheduleActionsTelemetry(logger: Logger, taskManager: TaskManagerStartContract) { + scheduleTasks(logger, taskManager); +} + +function registerActionsTelemetryTask( + logger: Logger, + taskManager: TaskManagerSetupContract, + core: CoreSetup, + kibanaIndex: string +) { + taskManager.registerTaskDefinitions({ + [TELEMETRY_TASK_TYPE]: { + title: 'Actions telemetry fetch task', + type: TELEMETRY_TASK_TYPE, + timeout: '5m', + createTaskRunner: telemetryTaskRunner(logger, core, kibanaIndex), + }, + }); +} + +async function scheduleTasks(logger: Logger, taskManager: TaskManagerStartContract) { + try { + await taskManager.ensureScheduled({ + id: TASK_ID, + taskType: TELEMETRY_TASK_TYPE, + state: { byDate: {}, suggestionsByDate: {}, saved: {}, runs: 0 }, + params: {}, + }); + } catch (e) { + logger.debug(`Error scheduling task, received ${e.message}`); + } +} + +export function telemetryTaskRunner(logger: Logger, core: CoreSetup, kibanaIndex: string) { + return ({ taskInstance }: RunContext) => { + const { state } = taskInstance; + const callCluster = core.elasticsearch.adminClient.callAsInternalUser; + return { + async run() { + return Promise.all([ + getTotalCount(callCluster, kibanaIndex), + getInUseTotalCount(callCluster, kibanaIndex), + ]) + .then(([totalAggegations, countActiveTotal]) => { + return { + state: { + runs: (state.runs || 0) + 1, + count_total: totalAggegations.countTotal, + count_by_type: totalAggegations.countByType, + count_active_total: countActiveTotal, + }, + runAt: getNextMidnight(), + }; + }) + .catch(errMsg => { + logger.warn(`Error executing actions telemetry task: ${errMsg}`); + return { + state: {}, + runAt: getNextMidnight(), + }; + }); + }, + }; + }; +} + +function getNextMidnight() { + return moment() + .add(1, 'd') + .startOf('d') + .toDate(); +} diff --git a/x-pack/plugins/actions/server/usage/types.ts b/x-pack/plugins/actions/server/usage/types.ts new file mode 100644 index 0000000000000..d1ea5b03f5415 --- /dev/null +++ b/x-pack/plugins/actions/server/usage/types.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface ActionsUsage { + count_total: number; + count_active_total: number; + count_by_type: Record; + count_active_by_type: Record; + // TODO: Implement executions count telemetry with eventLog, when it will write to index + // executions_by_type: Record; + // executions_total: number; +} diff --git a/x-pack/plugins/alerting/kibana.json b/x-pack/plugins/alerting/kibana.json index fb5003ede48ce..12f48d98dbf58 100644 --- a/x-pack/plugins/alerting/kibana.json +++ b/x-pack/plugins/alerting/kibana.json @@ -5,6 +5,6 @@ "kibanaVersion": "kibana", "configPath": ["xpack", "alerting"], "requiredPlugins": ["licensing", "taskManager", "encryptedSavedObjects", "actions"], - "optionalPlugins": ["spaces", "security"], + "optionalPlugins": ["usageCollection", "spaces", "security"], "ui": false } \ No newline at end of file diff --git a/x-pack/plugins/alerting/server/plugin.test.ts b/x-pack/plugins/alerting/server/plugin.test.ts index 40e620dd92af0..ec0ed4b761205 100644 --- a/x-pack/plugins/alerting/server/plugin.test.ts +++ b/x-pack/plugins/alerting/server/plugin.test.ts @@ -8,6 +8,7 @@ import { AlertingPlugin } from './plugin'; import { coreMock } from '../../../../src/core/server/mocks'; import { licensingMock } from '../../../plugins/licensing/server/mocks'; import { encryptedSavedObjectsMock } from '../../../plugins/encrypted_saved_objects/server/mocks'; +import { taskManagerMock } from '../../task_manager/server/mocks'; describe('Alerting Plugin', () => { describe('setup()', () => { @@ -28,6 +29,7 @@ describe('Alerting Plugin', () => { { licensing: licensingMock.createSetup(), encryptedSavedObjects: encryptedSavedObjectsSetup, + taskManager: taskManagerMock.createSetup(), } as any ); @@ -64,6 +66,7 @@ describe('Alerting Plugin', () => { { licensing: licensingMock.createSetup(), encryptedSavedObjects: encryptedSavedObjectsSetup, + taskManager: taskManagerMock.createSetup(), } as any ); @@ -105,6 +108,7 @@ describe('Alerting Plugin', () => { { licensing: licensingMock.createSetup(), encryptedSavedObjects: encryptedSavedObjectsSetup, + taskManager: taskManagerMock.createSetup(), } as any ); diff --git a/x-pack/plugins/alerting/server/plugin.ts b/x-pack/plugins/alerting/server/plugin.ts index bed163878b5ac..4d91c9b24add9 100644 --- a/x-pack/plugins/alerting/server/plugin.ts +++ b/x-pack/plugins/alerting/server/plugin.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { first, map } from 'rxjs/operators'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { SecurityPluginSetup } from '../../security/server'; import { EncryptedSavedObjectsPluginSetup, @@ -26,6 +28,7 @@ import { SavedObjectsServiceStart, IContextProvider, RequestHandler, + SharedGlobalConfig, } from '../../../../src/core/server'; import { @@ -50,6 +53,8 @@ import { PluginStartContract as ActionsPluginStartContract, } from '../../../plugins/actions/server'; import { Services } from './types'; +import { registerAlertsUsageCollector } from './usage'; +import { initializeAlertingTelemetry, scheduleAlertingTelemetry } from './usage/task'; export interface PluginSetupContract { registerType: AlertTypeRegistry['register']; @@ -66,6 +71,7 @@ export interface AlertingPluginsSetup { encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; licensing: LicensingPluginSetup; spaces?: SpacesPluginSetup; + usageCollection?: UsageCollectionSetup; } export interface AlertingPluginsStart { actions: ActionsPluginStartContract; @@ -84,11 +90,20 @@ export class AlertingPlugin { private spaces?: SpacesServiceSetup; private security?: SecurityPluginSetup; private readonly alertsClientFactory: AlertsClientFactory; + private readonly telemetryLogger: Logger; + private readonly kibanaIndex: Promise; constructor(initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get('plugins', 'alerting'); this.taskRunnerFactory = new TaskRunnerFactory(); this.alertsClientFactory = new AlertsClientFactory(); + this.telemetryLogger = initializerContext.logger.get('telemetry'); + this.kibanaIndex = initializerContext.config.legacy.globalConfig$ + .pipe( + first(), + map((config: SharedGlobalConfig) => config.kibana.index) + ) + .toPromise(); } public async setup(core: CoreSetup, plugins: AlertingPluginsSetup): Promise { @@ -124,6 +139,20 @@ export class AlertingPlugin { this.alertTypeRegistry = alertTypeRegistry; this.serverBasePath = core.http.basePath.serverBasePath; + const usageCollection = plugins.usageCollection; + if (usageCollection) { + core.getStartServices().then(async ([coreStart, startPlugins]: [CoreStart, any]) => { + registerAlertsUsageCollector(usageCollection, startPlugins.taskManager); + + initializeAlertingTelemetry( + this.telemetryLogger, + core, + plugins.taskManager, + await this.kibanaIndex + ); + }); + } + core.http.registerRouteHandlerContext('alerting', this.createRouteHandlerContext()); // Routes @@ -144,6 +173,16 @@ export class AlertingPlugin { muteAlertInstanceRoute(router, this.licenseState); unmuteAlertInstanceRoute(router, this.licenseState); + alertTypeRegistry.register({ + id: 'test', + actionGroups: [{ id: 'default', name: 'Default' }], + defaultActionGroupId: 'default', + name: 'Test', + executor: async options => { + return { status: 'ok' }; + }, + }); + return { registerType: alertTypeRegistry.register.bind(alertTypeRegistry), }; @@ -181,6 +220,8 @@ export class AlertingPlugin { getBasePath: this.getBasePath, }); + scheduleAlertingTelemetry(this.telemetryLogger, plugins.taskManager); + return { listTypes: alertTypeRegistry!.list.bind(this.alertTypeRegistry!), // Ability to get an alerts client from legacy code diff --git a/x-pack/plugins/alerting/server/usage/alerts_telemetry.ts b/x-pack/plugins/alerting/server/usage/alerts_telemetry.ts new file mode 100644 index 0000000000000..9ab63b7755500 --- /dev/null +++ b/x-pack/plugins/alerting/server/usage/alerts_telemetry.ts @@ -0,0 +1,305 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { APICaller } from 'kibana/server'; + +const alertTypeMetric = { + scripted_metric: { + init_script: 'state.types = [:]', + map_script: ` + String alertType = doc['alert.alertTypeId'].value; + state.types.put(alertType, state.types.containsKey(alertType) ? state.types.get(alertType) + 1 : 1); + `, + // Combine script is executed per cluster, but we already have a key-value pair per cluster. + // Despite docs that say this is optional, this script can't be blank. + combine_script: 'return state', + // Reduce script is executed across all clusters, so we need to add up all the total from each cluster + // This also needs to account for having no data + reduce_script: ` + Map result = [:]; + for (Map m : states.toArray()) { + if (m !== null) { + for (String k : m.keySet()) { + result.put(k, result.containsKey(k) ? result.get(k) + m.get(k) : m.get(k)); + } + } + } + return result; + `, + }, +}; + +export async function getTotalCountAggregations(callCluster: APICaller, kibanaInex: string) { + const throttleTimeMetric = { + scripted_metric: { + init_script: 'state.min = 0; state.max = 0; state.totalSum = 0; state.totalCount = 0;', + map_script: ` + if (doc['alert.throttle'].size() > 0) { + def throttle = doc['alert.throttle'].value; + + if (throttle.length() > 1) { + // get last char + String timeChar = throttle.substring(throttle.length() - 1); + // remove last char + throttle = throttle.substring(0, throttle.length() - 1); + + if (throttle.chars().allMatch(Character::isDigit)) { + // using of regex is not allowed in painless language + int parsed = Integer.parseInt(throttle); + + if (timeChar.equals("s")) { + parsed = parsed; + } else if (timeChar.equals("m")) { + parsed = parsed * 60; + } else if (timeChar.equals("h")) { + parsed = parsed * 60 * 60; + } else if (timeChar.equals("d")) { + parsed = parsed * 24 * 60 * 60; + } + if (state.min === 0 || parsed < state.min) { + state.min = parsed; + } + if (parsed > state.max) { + state.max = parsed; + } + state.totalSum += parsed; + state.totalCount++; + } + } + } + `, + // Combine script is executed per cluster, but we already have a key-value pair per cluster. + // Despite docs that say this is optional, this script can't be blank. + combine_script: 'return state', + // Reduce script is executed across all clusters, so we need to add up all the total from each cluster + // This also needs to account for having no data + reduce_script: ` + double min = 0; + double max = 0; + long totalSum = 0; + long totalCount = 0; + for (Map m : states.toArray()) { + if (m !== null) { + min = min > 0 ? Math.min(min, m.min) : m.min; + max = Math.max(max, m.max); + totalSum += m.totalSum; + totalCount += m.totalCount; + } + } + Map result = new HashMap(); + result.min = min; + result.max = max; + result.totalSum = totalSum; + result.totalCount = totalCount; + return result; + `, + }, + }; + + const intervalTimeMetric = { + scripted_metric: { + init_script: 'state.min = 0; state.max = 0; state.totalSum = 0; state.totalCount = 0;', + map_script: ` + if (doc['alert.schedule.interval'].size() > 0) { + def interval = doc['alert.schedule.interval'].value; + + if (interval.length() > 1) { + // get last char + String timeChar = interval.substring(interval.length() - 1); + // remove last char + interval = interval.substring(0, interval.length() - 1); + + if (interval.chars().allMatch(Character::isDigit)) { + // using of regex is not allowed in painless language + int parsed = Integer.parseInt(interval); + + if (timeChar.equals("s")) { + parsed = parsed; + } else if (timeChar.equals("m")) { + parsed = parsed * 60; + } else if (timeChar.equals("h")) { + parsed = parsed * 60 * 60; + } else if (timeChar.equals("d")) { + parsed = parsed * 24 * 60 * 60; + } + if (state.min === 0 || parsed < state.min) { + state.min = parsed; + } + if (parsed > state.max) { + state.max = parsed; + } + state.totalSum += parsed; + state.totalCount++; + } + } + } + `, + // Combine script is executed per cluster, but we already have a key-value pair per cluster. + // Despite docs that say this is optional, this script can't be blank. + combine_script: 'return state', + // Reduce script is executed across all clusters, so we need to add up all the total from each cluster + // This also needs to account for having no data + reduce_script: ` + double min = 0; + double max = 0; + long totalSum = 0; + long totalCount = 0; + for (Map m : states.toArray()) { + if (m !== null) { + min = min > 0 ? Math.min(min, m.min) : m.min; + max = Math.max(max, m.max); + totalSum += m.totalSum; + totalCount += m.totalCount; + } + } + Map result = new HashMap(); + result.min = min; + result.max = max; + result.totalSum = totalSum; + result.totalCount = totalCount; + return result; + `, + }, + }; + + const connectorsMetric = { + scripted_metric: { + init_script: + 'state.currentAlertActions = 0; state.min = 0; state.max = 0; state.totalActionsCount = 0;', + map_script: ` + String refName = doc['alert.actions.actionRef'].value; + if (refName == 'action_0') { + if (state.currentAlertActions !== 0 && state.currentAlertActions < state.min) { + state.min = state.currentAlertActions; + } + if (state.currentAlertActions !== 0 && state.currentAlertActions > state.max) { + state.max = state.currentAlertActions; + } + state.currentAlertActions = 1; + } else { + state.currentAlertActions++; + } + state.totalActionsCount++; + `, + // Combine script is executed per cluster, but we already have a key-value pair per cluster. + // Despite docs that say this is optional, this script can't be blank. + combine_script: 'return state', + // Reduce script is executed across all clusters, so we need to add up all the total from each cluster + // This also needs to account for having no data + reduce_script: ` + double min = 0; + double max = 0; + long totalActionsCount = 0; + long currentAlertActions = 0; + for (Map m : states.toArray()) { + if (m !== null) { + min = min > 0 ? Math.min(min, m.min) : m.min; + max = Math.max(max, m.max); + currentAlertActions += m.currentAlertActions; + totalActionsCount += m.totalActionsCount; + } + } + Map result = new HashMap(); + result.min = min; + result.max = max; + result.currentAlertActions = currentAlertActions; + result.totalActionsCount = totalActionsCount; + return result; + `, + }, + }; + + const results = await callCluster('search', { + index: kibanaInex, + rest_total_hits_as_int: true, + body: { + query: { + bool: { + filter: [{ term: { type: 'alert' } }], + }, + }, + aggs: { + byAlertTypeId: alertTypeMetric, + throttleTime: throttleTimeMetric, + intervalTime: intervalTimeMetric, + connectorsAgg: { + nested: { + path: 'alert.actions', + }, + aggs: { + connectors: connectorsMetric, + }, + }, + }, + }, + }); + + const totalAlertsCount = Object.keys(results.aggregations.byAlertTypeId.value.types).reduce( + (total: number, key: string) => + parseInt(results.aggregations.byAlertTypeId.value.types[key], 0) + total, + 0 + ); + + return { + count_total: totalAlertsCount, + count_by_type: results.aggregations.byAlertTypeId.value.types, + throttle_time: { + min: `${results.aggregations.throttleTime.value.min}s`, + avg: `${ + results.aggregations.throttleTime.value.totalCount > 0 + ? results.aggregations.throttleTime.value.totalSum / + results.aggregations.throttleTime.value.totalCount + : 0 + }s`, + max: `${results.aggregations.throttleTime.value.max}s`, + }, + schedule_time: { + min: `${results.aggregations.intervalTime.value.min}s`, + avg: `${ + results.aggregations.intervalTime.value.totalCount > 0 + ? results.aggregations.intervalTime.value.totalSum / + results.aggregations.intervalTime.value.totalCount + : 0 + }s`, + max: `${results.aggregations.intervalTime.value.max}s`, + }, + connectors_per_alert: { + min: results.aggregations.connectorsAgg.connectors.value.min, + avg: + totalAlertsCount > 0 + ? results.aggregations.connectorsAgg.connectors.value.totalActionsCount / totalAlertsCount + : 0, + max: results.aggregations.connectorsAgg.connectors.value.max, + }, + }; +} + +export async function getTotalCountInUse(callCluster: APICaller, kibanaInex: string) { + const searchResult = await callCluster('search', { + index: kibanaInex, + rest_total_hits_as_int: true, + body: { + query: { + bool: { + filter: [{ term: { type: 'alert' } }, { term: { 'alert.enabled': true } }], + }, + }, + aggs: { + byAlertTypeId: alertTypeMetric, + }, + }, + }); + return { + countTotal: Object.keys(searchResult.aggregations.byAlertTypeId.value.types).reduce( + (total: number, key: string) => + parseInt(searchResult.aggregations.byAlertTypeId.value.types[key], 0) + total, + 0 + ), + countByType: searchResult.aggregations.byAlertTypeId.value.types, + }; +} + +// TODO: Implement executions count telemetry with eventLog, when it will write to index diff --git a/x-pack/plugins/alerting/server/usage/alerts_usage_collector.test.ts b/x-pack/plugins/alerting/server/usage/alerts_usage_collector.test.ts new file mode 100644 index 0000000000000..e530c7afeebdc --- /dev/null +++ b/x-pack/plugins/alerting/server/usage/alerts_usage_collector.test.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { registerAlertsUsageCollector } from './alerts_usage_collector'; +import { taskManagerMock } from '../../../task_manager/server/task_manager.mock'; +const taskManagerStart = taskManagerMock.start(); + +beforeEach(() => jest.resetAllMocks()); + +describe('registerAlertsUsageCollector', () => { + let usageCollectionMock: jest.Mocked; + + beforeEach(() => { + usageCollectionMock = ({ + makeUsageCollector: jest.fn(), + registerCollector: jest.fn(), + } as unknown) as jest.Mocked; + }); + + it('should call registerCollector', () => { + registerAlertsUsageCollector(usageCollectionMock, taskManagerStart); + expect(usageCollectionMock.registerCollector).toHaveBeenCalledTimes(1); + }); + + it('should call makeUsageCollector with type = alerts', () => { + registerAlertsUsageCollector(usageCollectionMock, taskManagerStart); + expect(usageCollectionMock.makeUsageCollector).toHaveBeenCalledTimes(1); + expect(usageCollectionMock.makeUsageCollector.mock.calls[0][0].type).toBe('alerts'); + }); +}); diff --git a/x-pack/plugins/alerting/server/usage/alerts_usage_collector.ts b/x-pack/plugins/alerting/server/usage/alerts_usage_collector.ts new file mode 100644 index 0000000000000..d2cef0f717e94 --- /dev/null +++ b/x-pack/plugins/alerting/server/usage/alerts_usage_collector.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { get } from 'lodash'; +import { TaskManagerStartContract } from '../../../task_manager/server'; +import { AlertsUsage } from './types'; + +export function createAlertsUsageCollector( + usageCollection: UsageCollectionSetup, + taskManager: TaskManagerStartContract +) { + return usageCollection.makeUsageCollector({ + type: 'alerts', + isReady: () => true, + fetch: async (): Promise => { + try { + const doc = await getLatestTaskState(await taskManager); + // get the accumulated state from the recurring task + const state: AlertsUsage = get(doc, 'state'); + + return { + ...state, + }; + } catch (err) { + return { + count_total: 0, + count_active_total: 0, + count_disabled_total: 0, + throttle_time: { + min: 0, + avg: 0, + max: 0, + }, + schedule_time: { + min: 0, + avg: 0, + max: 0, + }, + connectors_per_alert: { + min: 0, + avg: 0, + max: 0, + }, + count_active_by_type: {}, + count_by_type: {}, + }; + } + }, + }); +} + +async function getLatestTaskState(taskManager: TaskManagerStartContract) { + try { + const result = await taskManager.get('Alerting-alerting_telemetry'); + return result; + } catch (err) { + const errMessage = err && err.message ? err.message : err.toString(); + /* + The usage service WILL to try to fetch from this collector before the task manager has been initialized, because the + task manager has to wait for all plugins to initialize first. It's fine to ignore it as next time around it will be + initialized (or it will throw a different type of error) + */ + if (!errMessage.includes('NotInitialized')) { + throw err; + } + } + + return null; +} + +export function registerAlertsUsageCollector( + usageCollection: UsageCollectionSetup, + taskManager: TaskManagerStartContract +) { + const collector = createAlertsUsageCollector(usageCollection, taskManager); + usageCollection.registerCollector(collector); +} diff --git a/x-pack/plugins/alerting/server/usage/index.ts b/x-pack/plugins/alerting/server/usage/index.ts new file mode 100644 index 0000000000000..c54900ebce09f --- /dev/null +++ b/x-pack/plugins/alerting/server/usage/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { registerAlertsUsageCollector } from './alerts_usage_collector'; diff --git a/x-pack/plugins/alerting/server/usage/task.ts b/x-pack/plugins/alerting/server/usage/task.ts new file mode 100644 index 0000000000000..3da60aef301e2 --- /dev/null +++ b/x-pack/plugins/alerting/server/usage/task.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger, CoreSetup } from 'kibana/server'; +import moment from 'moment'; +import { + RunContext, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '../../../task_manager/server'; + +import { getTotalCountAggregations, getTotalCountInUse } from './alerts_telemetry'; + +export const TELEMETRY_TASK_TYPE = 'alerting_telemetry'; + +export const TASK_ID = `Alerting-${TELEMETRY_TASK_TYPE}`; + +export function initializeAlertingTelemetry( + logger: Logger, + core: CoreSetup, + taskManager: TaskManagerSetupContract, + kibanaIndex: string +) { + registerAlertingTelemetryTask(logger, core, taskManager, kibanaIndex); +} + +export function scheduleAlertingTelemetry(logger: Logger, taskManager?: TaskManagerStartContract) { + if (taskManager) { + scheduleTasks(logger, taskManager); + } +} + +function registerAlertingTelemetryTask( + logger: Logger, + core: CoreSetup, + taskManager: TaskManagerSetupContract, + kibanaIndex: string +) { + taskManager.registerTaskDefinitions({ + [TELEMETRY_TASK_TYPE]: { + title: 'Alerting telemetry fetch task', + type: TELEMETRY_TASK_TYPE, + timeout: '5m', + createTaskRunner: telemetryTaskRunner(logger, core, kibanaIndex), + }, + }); +} + +async function scheduleTasks(logger: Logger, taskManager: TaskManagerStartContract) { + try { + await taskManager.ensureScheduled({ + id: TASK_ID, + taskType: TELEMETRY_TASK_TYPE, + state: { byDate: {}, suggestionsByDate: {}, saved: {}, runs: 0 }, + params: {}, + }); + } catch (e) { + logger.debug(`Error scheduling task, received ${e.message}`); + } +} + +export function telemetryTaskRunner(logger: Logger, core: CoreSetup, kibanaIndex: string) { + return ({ taskInstance }: RunContext) => { + const { state } = taskInstance; + const callCluster = core.elasticsearch.adminClient.callAsInternalUser; + return { + async run() { + return Promise.all([ + getTotalCountAggregations(callCluster, kibanaIndex), + getTotalCountInUse(callCluster, kibanaIndex), + ]) + .then(([totalCountAggregations, totalInUse]) => { + return { + state: { + runs: (state.runs || 0) + 1, + ...totalCountAggregations, + count_active_by_type: totalInUse.countByType, + count_active_total: totalInUse.countTotal, + count_disabled_total: totalCountAggregations.count_total - totalInUse.countTotal, + }, + runAt: getNextMidnight(), + }; + }) + .catch(errMsg => { + logger.warn(`Error executing alerting telemetry task: ${errMsg}`); + return { + state: {}, + runAt: getNextMidnight(), + }; + }); + }, + }; + }; +} + +function getNextMidnight() { + return moment() + .add(1, 'd') + .startOf('d') + .toDate(); +} diff --git a/x-pack/plugins/alerting/server/usage/types.ts b/x-pack/plugins/alerting/server/usage/types.ts new file mode 100644 index 0000000000000..71edefd336212 --- /dev/null +++ b/x-pack/plugins/alerting/server/usage/types.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface AlertsUsage { + count_total: number; + count_active_total: number; + count_disabled_total: number; + count_by_type: Record; + count_active_by_type: Record; + throttle_time: { + min: number; + avg: number; + max: number; + }; + schedule_time: { + min: number; + avg: number; + max: number; + }; + connectors_per_alert: { + min: number; + avg: number; + max: number; + }; +} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx index 9bdad54f03352..865ab6ea04cea 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import * as React from 'react'; -import { mountWithIntl } from 'test_utils/enzyme_helpers'; +import { mountWithIntl, nextTick } from 'test_utils/enzyme_helpers'; import { coreMock } from '../../../../../../../../src/core/public/mocks'; import { ReactWrapper } from 'enzyme'; import { act } from 'react-dom/test-utils'; @@ -49,7 +49,7 @@ actionTypeRegistry.list.mockReturnValue([]); describe('alerts_list component empty', () => { let wrapper: ReactWrapper; - beforeEach(async () => { + async function setup() { const { loadAlerts, loadAlertTypes } = jest.requireMock('../../../lib/alert_api'); const { loadActionTypes, loadAllActions } = jest.requireMock( '../../../lib/action_connector_api' @@ -114,22 +114,25 @@ describe('alerts_list component empty', () => { alertTypeRegistry: alertTypeRegistry as any, }; + wrapper = mountWithIntl( + + + + ); + await act(async () => { - wrapper = mountWithIntl( - - - - ); + await nextTick(); + wrapper.update(); }); + } - await waitForRender(wrapper); - }); - - it('renders empty list', () => { + it('renders empty list', async () => { + await setup(); expect(wrapper.find('[data-test-subj="createFirstAlertEmptyPrompt"]').exists()).toBeTruthy(); }); - it('renders Create alert button', () => { + it('renders Create alert button', async () => { + await setup(); expect( wrapper.find('[data-test-subj="createFirstAlertButton"]').find('EuiButton') ).toHaveLength(1); @@ -140,7 +143,7 @@ describe('alerts_list component empty', () => { describe('alerts_list component with items', () => { let wrapper: ReactWrapper; - beforeEach(async () => { + async function setup() { const { loadAlerts, loadAlertTypes } = jest.requireMock('../../../lib/alert_api'); const { loadActionTypes, loadAllActions } = jest.requireMock( '../../../lib/action_connector_api' @@ -239,21 +242,23 @@ describe('alerts_list component with items', () => { alertTypeRegistry: alertTypeRegistry as any, }; + wrapper = mountWithIntl( + + + + ); + await act(async () => { - wrapper = mountWithIntl( - - - - ); + await nextTick(); + wrapper.update(); }); - await waitForRender(wrapper); - expect(loadAlerts).toHaveBeenCalled(); expect(loadActionTypes).toHaveBeenCalled(); - }); + } - it('renders table of connectors', () => { + it('renders table of connectors', async () => { + await setup(); expect(wrapper.find('EuiBasicTable')).toHaveLength(1); expect(wrapper.find('EuiTableRow')).toHaveLength(2); }); @@ -262,7 +267,7 @@ describe('alerts_list component with items', () => { describe('alerts_list component empty with show only capability', () => { let wrapper: ReactWrapper; - beforeEach(async () => { + async function setup() { const { loadAlerts, loadAlertTypes } = jest.requireMock('../../../lib/alert_api'); const { loadActionTypes, loadAllActions } = jest.requireMock( '../../../lib/action_connector_api' @@ -330,18 +335,20 @@ describe('alerts_list component empty with show only capability', () => { alertTypeRegistry: {} as any, }; + wrapper = mountWithIntl( + + + + ); + await act(async () => { - wrapper = mountWithIntl( - - - - ); + await nextTick(); + wrapper.update(); }); + } - await waitForRender(wrapper); - }); - - it('not renders create alert button', () => { + it('not renders create alert button', async () => { + await setup(); expect(wrapper.find('[data-test-subj="createAlertButton"]')).toHaveLength(0); }); }); @@ -349,7 +356,7 @@ describe('alerts_list component empty with show only capability', () => { describe('alerts_list with show only capability', () => { let wrapper: ReactWrapper; - beforeEach(async () => { + async function setup() { const { loadAlerts, loadAlertTypes } = jest.requireMock('../../../lib/alert_api'); const { loadActionTypes, loadAllActions } = jest.requireMock( '../../../lib/action_connector_api' @@ -448,26 +455,22 @@ describe('alerts_list with show only capability', () => { alertTypeRegistry: alertTypeRegistry as any, }; + wrapper = mountWithIntl( + + + + ); + await act(async () => { - wrapper = mountWithIntl( - - - - ); + await nextTick(); + wrapper.update(); }); + } - await waitForRender(wrapper); - }); - - it('renders table of alerts with delete button disabled', () => { + it('renders table of alerts with delete button disabled', async () => { + await setup(); expect(wrapper.find('EuiBasicTable')).toHaveLength(1); expect(wrapper.find('EuiTableRow')).toHaveLength(2); // TODO: check delete button }); }); - -async function waitForRender(wrapper: ReactWrapper) { - await Promise.resolve(); - await Promise.resolve(); - wrapper.update(); -}