Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Monitoring] Update UI code to allow us to disable xpack.enabled in MB monitoring modules #72952

Closed
wants to merge 19 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/plugins/telemetry_collection_manager/server/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ export class TelemetryCollectionManagerPlugin
statsGetter,
clusterDetailsGetter,
licenseGetter,
customContext,
} = collectionConfig;

if (typeof priority !== 'number') {
Expand Down Expand Up @@ -118,6 +119,7 @@ export class TelemetryCollectionManagerPlugin
clusterDetailsGetter,
esCluster,
title,
customContext,
});
this.usageGetterMethodPriority = priority;
}
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugins/monitoring/server/alerts/base_alert.ts
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ export class BaseAlert {
const availableCcs = this.config.ui.ccs.enabled ? await fetchAvailableCcs(callCluster) : [];
// Support CCS use cases by querying to find available remote clusters
// and then adding those to the index pattern we are searching against
let esIndexPattern = INDEX_PATTERN_ELASTICSEARCH;
let esIndexPattern = `${INDEX_PATTERN_ELASTICSEARCH},${this.config.ui.metricbeat.index}`;
if (availableCcs) {
esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,11 @@ describe('ClusterHealthAlert', () => {
});
const monitoringCluster = null;
const config = {
ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
ui: {
ccs: { enabled: true },
container: { elasticsearch: { enabled: false } },
metricbeat: { index: 'metricbeat-* ' },
},
};
const kibanaUrl = 'http://localhost:5601';

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,11 @@ describe('CpuUsageAlert', () => {
});
const monitoringCluster = null;
const config = {
ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
ui: {
ccs: { enabled: true },
container: { elasticsearch: { enabled: false } },
metricbeat: { index: 'metricbeat-* ' },
},
};
const kibanaUrl = 'http://localhost:5601';

Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugins/monitoring/server/alerts/cpu_usage_alert.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ export class CpuUsageAlert extends BaseAlert {
uiSettings: IUiSettingsClient,
availableCcs: string[]
): Promise<AlertData[]> {
let esIndexPattern = INDEX_PATTERN_ELASTICSEARCH;
let esIndexPattern = `${INDEX_PATTERN_ELASTICSEARCH},${this.config.ui.metricbeat.index}`;
if (availableCcs) {
esIndexPattern = getCcsIndexPattern(esIndexPattern, availableCcs);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,11 @@ describe('ElasticsearchVersionMismatchAlert', () => {
});
const monitoringCluster = null;
const config = {
ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
ui: {
ccs: { enabled: true },
container: { elasticsearch: { enabled: false } },
metricbeat: { index: 'metricbeat-* ' },
},
};
const kibanaUrl = 'http://localhost:5601';

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,11 @@ describe('KibanaVersionMismatchAlert', () => {
});
const monitoringCluster = null;
const config = {
ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
ui: {
ccs: { enabled: true },
container: { elasticsearch: { enabled: false } },
metricbeat: { index: 'metricbeat-* ' },
},
};
const kibanaUrl = 'http://localhost:5601';

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,11 @@ describe('LicenseExpirationAlert', () => {
});
const monitoringCluster = null;
const config = {
ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
ui: {
ccs: { enabled: true },
container: { elasticsearch: { enabled: false } },
metricbeat: { index: 'metricbeat-* ' },
},
};
const kibanaUrl = 'http://localhost:5601';

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,11 @@ describe('LogstashVersionMismatchAlert', () => {
});
const monitoringCluster = null;
const config = {
ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
ui: {
ccs: { enabled: true },
container: { elasticsearch: { enabled: false } },
metricbeat: { index: 'metricbeat-* ' },
},
};
const kibanaUrl = 'http://localhost:5601';

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,11 @@ describe('NodesChangedAlert', () => {
});
const monitoringCluster = null;
const config = {
ui: { ccs: { enabled: true }, container: { elasticsearch: { enabled: false } } },
ui: {
ccs: { enabled: true },
container: { elasticsearch: { enabled: false } },
metricbeat: { index: 'metricbeat-* ' },
},
};
const kibanaUrl = 'http://localhost:5601';

Expand Down
3 changes: 3 additions & 0 deletions x-pack/plugins/monitoring/server/config.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@ describe('config schema', () => {
"index": "filebeat-*",
},
"max_bucket_size": 10000,
"metricbeat": Object {
"index": "metricbeat-*",
},
"min_interval_seconds": 10,
"show_license_expiration": true,
},
Expand Down
3 changes: 3 additions & 0 deletions x-pack/plugins/monitoring/server/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ export const configSchema = schema.object({
logs: schema.object({
index: schema.string({ defaultValue: 'filebeat-*' }),
}),
metricbeat: schema.object({
index: schema.string({ defaultValue: 'metricbeat-*' }),
}),
max_bucket_size: schema.number({ defaultValue: 10000 }),
elasticsearch: monitoringElasticsearchConfigSchema,
container: schema.object({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export async function fetchClusters(callCluster: any, index: string): Promise<Al
filterPath: [
'hits.hits._source.cluster_settings.cluster.metadata.display_name',
'hits.hits._source.cluster_uuid',
'hits.hits._source.elasticsearch.cluster.id',
'hits.hits._source.cluster_name',
],
body: {
Expand Down Expand Up @@ -45,9 +46,10 @@ export async function fetchClusters(callCluster: any, index: string): Promise<Al
const clusterName: string =
get(hit, '_source.cluster_settings.cluster.metadata.display_name') ||
get(hit, '_source.cluster_name') ||
get(hit, '_source.elasticsearch.cluster.id') ||
get(hit, '_source.cluster_uuid');
return {
clusterUuid: get(hit, '_source.cluster_uuid'),
clusterUuid: get(hit, '_source.elasticsearch.cluster.id', get(hit, '_source.cluster_uuid')),
clusterName,
};
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,9 @@ export async function getTimeOfLastEvent({
};

const response = await callWithRequest(req, 'search', params);
return get(response, 'hits.hits[0]._source.timestamp');
return get(
response,
'hits.hits[0]._source.@timestamp',
get(response, 'hits.hits[0]._source.timestamp')
);
}
24 changes: 22 additions & 2 deletions x-pack/plugins/monitoring/server/lib/apm/get_apm_info.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,14 @@ import { getTimeOfLastEvent } from './_get_time_of_last_event';
export function handleResponse(response, apmUuid) {
const firstStats = get(
response,
'hits.hits[0].inner_hits.first_hit.hits.hits[0]._source.beats_stats'
'hits.hits[0].inner_hits.first_hit.hits.hits[0]._source.beats.stats',
get(response, 'hits.hits[0].inner_hits.first_hit.hits.hits[0]._source.beats_stats')
);
const stats = get(
response,
'hits.hits[0]._source.beats.stats',
get(response, 'hits.hits[0]._source.beats_stats')
);
const stats = get(response, 'hits.hits[0]._source.beats_stats');

const eventsTotalFirst = get(firstStats, 'metrics.libbeat.pipeline.events.total', null);
const eventsEmittedFirst = get(firstStats, 'metrics.libbeat.pipeline.events.published', null);
Expand Down Expand Up @@ -57,20 +62,35 @@ export async function getApmInfo(req, apmIndexPattern, { clusterUuid, apmUuid, s
ignoreUnavailable: true,
filterPath: [
'hits.hits._source.beats_stats.beat.host',
'hits.hits._source.beats.stats.beat.host',
'hits.hits._source.beats_stats.beat.version',
'hits.hits._source.beats.stats.beat.version',
'hits.hits._source.beats_stats.beat.name',
'hits.hits._source.beats.stats.beat.name',
'hits.hits._source.beats_stats.beat.type',
'hits.hits._source.beats.stats.beat.type',
'hits.hits._source.beats_stats.metrics.libbeat.output.type',
'hits.hits._source.beats.stats.metrics.libbeat.output.type',
'hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.published',
'hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.published',
'hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.total',
'hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.total',
'hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.dropped',
'hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.dropped',
'hits.hits._source.beats_stats.metrics.libbeat.output.write.bytes',
'hits.hits._source.beats.stats.metrics.libbeat.output.write.bytes',
'hits.hits._source.beats_stats.metrics.libbeat.config.reloads',
'hits.hits._source.beats.stats.metrics.libbeat.config.reloads',
'hits.hits._source.beats_stats.metrics.beat.info.uptime.ms',
'hits.hits._source.beats.stats.metrics.beat.info.uptime.ms',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.published',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.published',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.total',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.total',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.dropped',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.dropped',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats_stats.metrics.libbeat.output.write.bytes',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats.stats.metrics.libbeat.output.write.bytes',
],
body: {
sort: { timestamp: { order: 'desc' } },
Expand Down
28 changes: 25 additions & 3 deletions x-pack/plugins/monitoring/server/lib/apm/get_apms.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export function handleResponse(response, start, end) {
const hits = get(response, 'hits.hits', []);
const initial = { ids: new Set(), beats: [] };
const { beats } = hits.reduce((accum, hit) => {
const stats = get(hit, '_source.beats_stats');
const stats = get(hit, '_source.beats.stats', get(hit, '_source.beats_stats'));
const uuid = get(stats, 'beat.uuid');

// skip this duplicated beat, newer one was already added
Expand All @@ -25,7 +25,11 @@ export function handleResponse(response, start, end) {

// add another beat summary
accum.ids.add(uuid);
const earliestStats = get(hit, 'inner_hits.earliest.hits.hits[0]._source.beats_stats');
const earliestStats = get(
hit,
'inner_hits.earliest.hits.hits[0]._source.beats.stats',
get(hit, 'inner_hits.earliest.hits.hits[0]._source.beats_stats')
);

// add the beat
const rateOptions = {
Expand Down Expand Up @@ -66,7 +70,7 @@ export function handleResponse(response, start, end) {
errors,
memory: get(stats, 'metrics.beat.memstats.memory_alloc'),
version: get(stats, 'beat.version'),
time_of_last_event: get(hit, '_source.timestamp'),
time_of_last_event: get(hit, '_source.@timestamp', get(hit, '_source.timestamp')),
});

return accum;
Expand All @@ -89,29 +93,47 @@ export async function getApms(req, apmIndexPattern, clusterUuid) {
filterPath: [
// only filter path can filter for inner_hits
'hits.hits._source.timestamp',
'hits.hits._source.@timestamp',
'hits.hits._source.beats_stats.beat.uuid',
'hits.hits._source.beats.stats.beat.uuid',
'hits.hits._source.beats_stats.beat.name',
'hits.hits._source.beats.stats.beat.name',
'hits.hits._source.beats_stats.beat.host',
'hits.hits._source.beats.stats.beat.host',
'hits.hits._source.beats_stats.beat.type',
'hits.hits._source.beats.stats.beat.type',
'hits.hits._source.beats_stats.beat.version',
'hits.hits._source.beats.stats.beat.version',
'hits.hits._source.beats_stats.metrics.libbeat.output.type',
'hits.hits._source.beats.stats.metrics.libbeat.output.type',
'hits.hits._source.beats_stats.metrics.libbeat.output.read.errors',
'hits.hits._source.beats.stats.metrics.libbeat.output.read.errors',
'hits.hits._source.beats_stats.metrics.libbeat.output.write.errors',
'hits.hits._source.beats.stats.metrics.libbeat.output.write.errors',
'hits.hits._source.beats_stats.metrics.beat.memstats.memory_alloc',
'hits.hits._source.beats.stats.metrics.beat.memstats.memory_alloc',

// latest hits for calculating metrics
'hits.hits._source.beats_stats.timestamp',
'hits.hits._source.beats.stats.timestamp',
'hits.hits._source.beats_stats.metrics.libbeat.output.write.bytes',
'hits.hits._source.beats.stats.metrics.libbeat.output.write.bytes',
'hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.total',
'hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.total',

// earliest hits for calculating metrics
'hits.hits.inner_hits.earliest.hits.hits._source.beats_stats.timestamp',
'hits.hits.inner_hits.earliest.hits.hits._source.beats.stats.timestamp',
'hits.hits.inner_hits.earliest.hits.hits._source.beats_stats.metrics.libbeat.output.write.bytes',
'hits.hits.inner_hits.earliest.hits.hits._source.beats.stats.metrics.libbeat.output.write.bytes',
'hits.hits.inner_hits.earliest.hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.total',
'hits.hits.inner_hits.earliest.hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.total',

// earliest hits for calculating diffs
'hits.hits.inner_hits.earliest.hits.hits._source.beats_stats.metrics.libbeat.output.read.errors',
'hits.hits.inner_hits.earliest.hits.hits._source.beats.stats.metrics.libbeat.output.read.errors',
'hits.hits.inner_hits.earliest.hits.hits._source.beats_stats.metrics.libbeat.output.write.errors',
'hits.hits.inner_hits.earliest.hits.hits._source.beats.stats.metrics.libbeat.output.write.errors',
],
body: {
query: createApmQuery({
Expand Down
26 changes: 24 additions & 2 deletions x-pack/plugins/monitoring/server/lib/beats/get_beat_summary.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,14 @@ import { getDiffCalculation } from './_beats_stats';
export function handleResponse(response, beatUuid) {
const firstStats = get(
response,
'hits.hits[0].inner_hits.first_hit.hits.hits[0]._source.beats_stats'
'hits.hits[0].inner_hits.first_hit.hits.hits[0]._source.beats.stats',
get(response, 'hits.hits[0].inner_hits.first_hit.hits.hits[0]._source.beats_stats')
);
const stats = get(
response,
'hits.hits[0]._source.beats.stats',
get(response, 'hits.hits[0]._source.beats_stats')
);
const stats = get(response, 'hits.hits[0]._source.beats_stats');

const eventsTotalFirst = get(firstStats, 'metrics.libbeat.pipeline.events.total', null);
const eventsEmittedFirst = get(firstStats, 'metrics.libbeat.pipeline.events.published', null);
Expand Down Expand Up @@ -60,22 +65,39 @@ export async function getBeatSummary(
ignoreUnavailable: true,
filterPath: [
'hits.hits._source.beats_stats.beat.host',
'hits.hits._source.beats.stats.beat.host',
'hits.hits._source.beats_stats.beat.version',
'hits.hits._source.beats.stats.beat.version',
'hits.hits._source.beats_stats.beat.name',
'hits.hits._source.beats.stats.beat.name',
'hits.hits._source.beats_stats.beat.type',
'hits.hits._source.beats.stats.beat.type',
'hits.hits._source.beats_stats.metrics.libbeat.output.type',
'hits.hits._source.beats.stats.metrics.libbeat.output.type',
'hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.published',
'hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.published',
'hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.total',
'hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.total',
'hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.dropped',
'hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.dropped',
'hits.hits._source.beats_stats.metrics.libbeat.output.write.bytes',
'hits.hits._source.beats.stats.metrics.libbeat.output.write.bytes',
'hits.hits._source.beats_stats.metrics.libbeat.config.reloads',
'hits.hits._source.beats.stats.metrics.libbeat.config.reloads',
'hits.hits._source.beats_stats.metrics.beat.info.uptime.ms',
'hits.hits._source.beats.stats.metrics.beat.info.uptime.ms',
'hits.hits._source.beats_stats.metrics.beat.handles.limit.hard',
'hits.hits._source.beats.stats.metrics.beat.handles.limit.hard',
'hits.hits._source.beats_stats.metrics.beat.handles.limit.soft',
'hits.hits._source.beats.stats.metrics.beat.handles.limit.soft',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.published',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.published',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.total',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.total',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats_stats.metrics.libbeat.pipeline.events.dropped',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats.stats.metrics.libbeat.pipeline.events.dropped',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats_stats.metrics.libbeat.output.write.bytes',
'hits.hits.inner_hits.first_hit.hits.hits._source.beats.stats.metrics.libbeat.output.write.bytes',
],
body: {
sort: { timestamp: { order: 'desc' } },
Expand Down
Loading