From cd792351fdba68686c7d9584f7ddac9c362addaf Mon Sep 17 00:00:00 2001 From: chrisronline Date: Wed, 18 Nov 2020 11:14:24 -0500 Subject: [PATCH 1/3] Only look at ES for the missing data alert for now --- .../missing_monitoring_data_alert.test.ts | 50 ++++--------------- .../alerts/missing_monitoring_data_alert.ts | 7 ++- 2 files changed, 17 insertions(+), 40 deletions(-) diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts index 57d01dc6a1100..bc6d65d7078c4 100644 --- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts @@ -140,7 +140,7 @@ describe('MissingMonitoringDataAlert', () => { // @ts-ignore params: alert.defaultParams, } as any); - const count = 2; + const count = 1; expect(replaceState).toHaveBeenCalledWith({ alertStates: [ { @@ -195,53 +195,25 @@ describe('MissingMonitoringDataAlert', () => { stackProductName: 'kibanaInstance1', stackProductUuid: 'kibanaUuid1', ui: { - isFiring: true, - message: { - text: - 'For the past an hour, we have not detected any monitoring data from the Kibana instance: kibanaInstance1, starting at #absolute', - nextSteps: [ - { - text: '#start_linkView all Kibana instances#end_link', - tokens: [ - { - startToken: '#start_link', - endToken: '#end_link', - type: 'link', - url: 'kibana/instances', - }, - ], - }, - { - text: 'Verify monitoring settings on the instance', - }, - ], - tokens: [ - { - startToken: '#absolute', - type: 'time', - isAbsolute: true, - isRelative: false, - timestamp: 1, - }, - ], - }, + isFiring: false, + message: null, severity: 'danger', resolvedMS: 0, - triggeredMS: 1, + triggeredMS: 0, lastCheckedMS: 0, }, }, ], }); expect(scheduleActions).toHaveBeenCalledWith('default', { - internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, - internalShortMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, + internalFullMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. [View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, + internalShortMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, actionPlain: 'Verify these stack products are up and running, then double check the monitoring settings.', clusterName, count, - stackProducts: 'Elasticsearch node: esName1, Kibana instance: kibanaInstance1', + stackProducts: 'Elasticsearch node: esName1', state: 'firing', }); }); @@ -442,16 +414,16 @@ describe('MissingMonitoringDataAlert', () => { // @ts-ignore params: alert.defaultParams, } as any); - const count = 2; + const count = 1; expect(scheduleActions).toHaveBeenCalledWith('default', { - internalFullMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, - internalShortMessage: `We have not detected any monitoring data for 2 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, + internalFullMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, + internalShortMessage: `We have not detected any monitoring data for 1 stack product(s) in cluster: testCluster. Verify these stack products are up and running, then double check the monitoring settings.`, action: `[View what monitoring data we do have for these stack products.](http://localhost:5601/app/monitoring#/overview?_g=(cluster_uuid:abc123))`, actionPlain: 'Verify these stack products are up and running, then double check the monitoring settings.', clusterName, count, - stackProducts: 'Elasticsearch node: esName1, Kibana instance: kibanaInstance1', + stackProducts: 'Elasticsearch node: esName1', state: 'firing', }); }); diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts index 4001c6b9b3ed2..adfb442c00756 100644 --- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts @@ -27,6 +27,7 @@ import { ALERT_MISSING_MONITORING_DATA, INDEX_PATTERN_ELASTICSEARCH, ALERT_DETAILS, + ELASTICSEARCH_SYSTEM_ID, } from '../../common/constants'; import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; import { AlertMessageTokenType, AlertSeverity } from '../../common/enums'; @@ -139,10 +140,14 @@ export class MissingMonitoringDataAlert extends BaseAlert { now - limit - LIMIT_BUFFER ); return missingData.map((missing) => { + // TODO: only Elasticsearch until we can figure out how to handle upgrades for the rest of the stack + // https://github.com/elastic/kibana/issues/83309 + const shouldFire = + missing.gapDuration > duration && missing.stackProduct === ELASTICSEARCH_SYSTEM_ID; return { instanceKey: `${missing.clusterUuid}:${missing.stackProduct}:${missing.stackProductUuid}`, clusterUuid: missing.clusterUuid, - shouldFire: missing.gapDuration > duration, + shouldFire, severity: AlertSeverity.Danger, meta: { missing, limit }, ccs: missing.ccs, From 5e05818827201e0249f351452b9b8ad1f8b3ce4c Mon Sep 17 00:00:00 2001 From: chrisronline Date: Thu, 19 Nov 2020 13:00:19 -0500 Subject: [PATCH 2/3] PR feedback --- .../alerts/missing_monitoring_data_alert.ts | 7 +- .../fetch_missing_monitoring_data.test.ts | 89 --------- .../alerts/fetch_missing_monitoring_data.ts | 178 +++--------------- 3 files changed, 28 insertions(+), 246 deletions(-) diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts index adfb442c00756..4001c6b9b3ed2 100644 --- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts +++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.ts @@ -27,7 +27,6 @@ import { ALERT_MISSING_MONITORING_DATA, INDEX_PATTERN_ELASTICSEARCH, ALERT_DETAILS, - ELASTICSEARCH_SYSTEM_ID, } from '../../common/constants'; import { getCcsIndexPattern } from '../lib/alerts/get_ccs_index_pattern'; import { AlertMessageTokenType, AlertSeverity } from '../../common/enums'; @@ -140,14 +139,10 @@ export class MissingMonitoringDataAlert extends BaseAlert { now - limit - LIMIT_BUFFER ); return missingData.map((missing) => { - // TODO: only Elasticsearch until we can figure out how to handle upgrades for the rest of the stack - // https://github.com/elastic/kibana/issues/83309 - const shouldFire = - missing.gapDuration > duration && missing.stackProduct === ELASTICSEARCH_SYSTEM_ID; return { instanceKey: `${missing.clusterUuid}:${missing.stackProduct}:${missing.stackProductUuid}`, clusterUuid: missing.clusterUuid, - shouldFire, + shouldFire: missing.gapDuration > duration, severity: AlertSeverity.Danger, meta: { missing, limit }, ccs: missing.ccs, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts index b09f5a88dba9c..7edd7496805a0 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.test.ts @@ -75,63 +75,6 @@ describe('fetchMissingMonitoringData', () => { timestamp: 2, }, ]), - kibana_uuids: getResponse('.monitoring-kibana-*', [ - { - uuid: 'kibanaUuid1', - nameSource: { - kibana_stats: { - kibana: { - name: 'kibanaName1', - }, - }, - }, - timestamp: 4, - }, - ]), - logstash_uuids: getResponse('.monitoring-logstash-*', [ - { - uuid: 'logstashUuid1', - nameSource: { - logstash_stats: { - logstash: { - host: 'logstashName1', - }, - }, - }, - timestamp: 2, - }, - ]), - beats: { - beats_uuids: getResponse('.monitoring-beats-*', [ - { - uuid: 'beatUuid1', - nameSource: { - beats_stats: { - beat: { - name: 'beatName1', - }, - }, - }, - timestamp: 0, - }, - ]), - }, - apms: { - apm_uuids: getResponse('.monitoring-beats-*', [ - { - uuid: 'apmUuid1', - nameSource: { - beats_stats: { - beat: { - name: 'apmName1', - type: 'apm-server', - }, - }, - }, - timestamp: 1, - }, - ]), - }, })), }, }, @@ -162,38 +105,6 @@ describe('fetchMissingMonitoringData', () => { gapDuration: 8, ccs: null, }, - { - stackProduct: 'kibana', - stackProductUuid: 'kibanaUuid1', - stackProductName: 'kibanaName1', - clusterUuid: 'clusterUuid1', - gapDuration: 6, - ccs: null, - }, - { - stackProduct: 'logstash', - stackProductUuid: 'logstashUuid1', - stackProductName: 'logstashName1', - clusterUuid: 'clusterUuid1', - gapDuration: 8, - ccs: null, - }, - { - stackProduct: 'beats', - stackProductUuid: 'beatUuid1', - stackProductName: 'beatName1', - clusterUuid: 'clusterUuid1', - gapDuration: 10, - ccs: null, - }, - { - stackProduct: 'apm', - stackProductUuid: 'apmUuid1', - stackProductName: 'apmName1', - clusterUuid: 'clusterUuid1', - gapDuration: 9, - ccs: null, - }, ]); }); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts index 49307764e9f01..b4e12e5d86139 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_missing_monitoring_data.ts @@ -5,25 +5,11 @@ */ import { get } from 'lodash'; import { AlertCluster, AlertMissingData } from '../../../common/types/alerts'; -import { - KIBANA_SYSTEM_ID, - BEATS_SYSTEM_ID, - APM_SYSTEM_ID, - LOGSTASH_SYSTEM_ID, - ELASTICSEARCH_SYSTEM_ID, -} from '../../../common/constants'; +import { ELASTICSEARCH_SYSTEM_ID } from '../../../common/constants'; interface ClusterBucketESResponse { key: string; - kibana_uuids?: UuidResponse; - logstash_uuids?: UuidResponse; - es_uuids?: UuidResponse; - beats?: { - beats_uuids: UuidResponse; - }; - apms?: { - apm_uuids: UuidResponse; - }; + es_uuids: UuidResponse; } interface UuidResponse { @@ -48,44 +34,11 @@ interface TopHitESResponse { source_node?: { name: string; }; - kibana_stats?: { - kibana: { - name: string; - }; - }; - logstash_stats?: { - logstash: { - host: string; - }; - }; - beats_stats?: { - beat: { - name: string; - type: string; - }; - }; }; } -function getStackProductFromIndex(index: string, beatType: string) { - if (index.includes('-kibana-')) { - return KIBANA_SYSTEM_ID; - } - if (index.includes('-beats-')) { - if (beatType === 'apm-server') { - return APM_SYSTEM_ID; - } - return BEATS_SYSTEM_ID; - } - if (index.includes('-logstash-')) { - return LOGSTASH_SYSTEM_ID; - } - if (index.includes('-es-')) { - return ELASTICSEARCH_SYSTEM_ID; - } - return ''; -} - +// TODO: only Elasticsearch until we can figure out how to handle upgrades for the rest of the stack +// https://github.com/elastic/kibana/issues/83309 export async function fetchMissingMonitoringData( callCluster: any, clusters: AlertCluster[], @@ -95,37 +48,6 @@ export async function fetchMissingMonitoringData( startMs: number ): Promise { const endMs = nowInMs; - - const nameFields = [ - 'source_node.name', - 'kibana_stats.kibana.name', - 'logstash_stats.logstash.host', - 'beats_stats.beat.name', - 'beat_stats.beat.type', - ]; - const subAggs = { - most_recent: { - max: { - field: 'timestamp', - }, - }, - document: { - top_hits: { - size: 1, - sort: [ - { - timestamp: { - order: 'desc', - }, - }, - ], - _source: { - includes: ['_index', ...nameFields], - }, - }, - }, - }; - const params = { index, filterPath: ['aggregations.clusters.buckets'], @@ -163,61 +85,28 @@ export async function fetchMissingMonitoringData( field: 'node_stats.node_id', size, }, - aggs: subAggs, - }, - kibana_uuids: { - terms: { - field: 'kibana_stats.kibana.uuid', - size, - }, - aggs: subAggs, - }, - beats: { - filter: { - bool: { - must_not: { - term: { - 'beats_stats.beat.type': 'apm-server', - }, - }, - }, - }, aggs: { - beats_uuids: { - terms: { - field: 'beats_stats.beat.uuid', - size, + most_recent: { + max: { + field: 'timestamp', }, - aggs: subAggs, }, - }, - }, - apms: { - filter: { - bool: { - must: { - term: { - 'beats_stats.beat.type': 'apm-server', + document: { + top_hits: { + size: 1, + sort: [ + { + timestamp: { + order: 'desc', + }, + }, + ], + _source: { + includes: ['_index', 'source_node.name'], }, }, }, }, - aggs: { - apm_uuids: { - terms: { - field: 'beats_stats.beat.uuid', - size, - }, - aggs: subAggs, - }, - }, - }, - logstash_uuids: { - terms: { - field: 'logstash_stats.logstash.uuid', - size, - }, - aggs: subAggs, }, }, }, @@ -234,33 +123,20 @@ export async function fetchMissingMonitoringData( const uniqueList: { [id: string]: AlertMissingData } = {}; for (const clusterBucket of clusterBuckets) { const clusterUuid = clusterBucket.key; - - const uuidBuckets = [ - ...(clusterBucket.es_uuids?.buckets || []), - ...(clusterBucket.kibana_uuids?.buckets || []), - ...(clusterBucket.logstash_uuids?.buckets || []), - ...(clusterBucket.beats?.beats_uuids.buckets || []), - ...(clusterBucket.apms?.apm_uuids.buckets || []), - ]; + const uuidBuckets = clusterBucket.es_uuids.buckets; for (const uuidBucket of uuidBuckets) { const stackProductUuid = uuidBucket.key; const indexName = get(uuidBucket, `document.hits.hits[0]._index`); - const stackProduct = getStackProductFromIndex( - indexName, - get(uuidBucket, `document.hits.hits[0]._source.beats_stats.beat.type`) - ); const differenceInMs = nowInMs - uuidBucket.most_recent.value; - let stackProductName = stackProductUuid; - for (const nameField of nameFields) { - stackProductName = get(uuidBucket, `document.hits.hits[0]._source.${nameField}`); - if (stackProductName) { - break; - } - } + const stackProductName = get( + uuidBucket, + `document.hits.hits[0]._source.source_node.name`, + stackProductUuid + ); - uniqueList[`${clusterUuid}${stackProduct}${stackProductUuid}`] = { - stackProduct, + uniqueList[`${clusterUuid}${stackProductUuid}`] = { + stackProduct: ELASTICSEARCH_SYSTEM_ID, stackProductUuid, stackProductName, clusterUuid, From e4ca151e9368fa977c5fb0f789a5c4629f52984c Mon Sep 17 00:00:00 2001 From: chrisronline Date: Mon, 23 Nov 2020 10:42:08 -0500 Subject: [PATCH 3/3] Fix tests --- .../missing_monitoring_data_alert.test.ts | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts index bc6d65d7078c4..1332148a61cdd 100644 --- a/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts +++ b/x-pack/plugins/monitoring/server/alerts/missing_monitoring_data_alert.test.ts @@ -65,13 +65,6 @@ describe('MissingMonitoringDataAlert', () => { clusterUuid, gapDuration, }, - { - stackProduct: 'kibana', - stackProductUuid: 'kibanaUuid1', - stackProductName: 'kibanaInstance1', - clusterUuid, - gapDuration: gapDuration + 10, - }, ]; const getUiSettingsService = () => ({ asScopedToClient: jest.fn(), @@ -187,22 +180,6 @@ describe('MissingMonitoringDataAlert', () => { lastCheckedMS: 0, }, }, - { - ccs: undefined, - cluster: { clusterUuid, clusterName }, - gapDuration: gapDuration + 10, - stackProduct: 'kibana', - stackProductName: 'kibanaInstance1', - stackProductUuid: 'kibanaUuid1', - ui: { - isFiring: false, - message: null, - severity: 'danger', - resolvedMS: 0, - triggeredMS: 0, - lastCheckedMS: 0, - }, - }, ], }); expect(scheduleActions).toHaveBeenCalledWith('default', {