diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/__snapshots__/ccr.test.js.snap b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/__snapshots__/ccr.test.js.snap new file mode 100644 index 0000000000000..80d38edee404c --- /dev/null +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/__snapshots__/ccr.test.js.snap @@ -0,0 +1,128 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Ccr that it renders normally 1`] = ` + + + + + + + + + +`; diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/ccr.css b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/ccr.css new file mode 100644 index 0000000000000..285eb066c6988 --- /dev/null +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/ccr.css @@ -0,0 +1,7 @@ +/** + * [1] - We want the collapsed table (that shows the shard data) to be inline + * with the columns from the main table so we need to remove the padding + */ +.monitoringElasticsearchCcrListingTable .euiTableRow-isExpandedRow > .euiTableRowCell > .euiTableCellContent { + padding: 0; /* [1] */ +} diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/ccr.js b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/ccr.js new file mode 100644 index 0000000000000..f57ba2caec380 --- /dev/null +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/ccr.js @@ -0,0 +1,211 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, Component } from 'react'; +import { + EuiInMemoryTable, + EuiLink, + EuiPage, + EuiPageBody, + EuiPageContent, + EuiPageContentBody, + EuiIcon, + EuiIconTip, + EuiTextColor +} from '@elastic/eui'; + +import './ccr.css'; + +function toSeconds(ms) { + return Math.floor(ms / 1000) + 's'; +} + +export class Ccr extends Component { + constructor(props) { + super(props); + this.state = { + itemIdToExpandedRowMap: {}, + }; + } + + toggleShards(index, shards) { + const itemIdToExpandedRowMap = { + ...this.state.itemIdToExpandedRowMap + }; + + if (itemIdToExpandedRowMap[index]) { + delete itemIdToExpandedRowMap[index]; + } else { + let pagination = { + initialPageSize: 5, + pageSizeOptions: [5, 10, 20] + }; + + if (shards.length <= pagination.initialPageSize) { + pagination = false; + } + + itemIdToExpandedRowMap[index] = ( + { + return ( + + {shardId} + + ); + } + }, + { + render: () => null + }, + { + field: 'opsSynced', + name: 'Ops synced' + }, + { + field: 'syncLagTime', + name: 'Last fetch time', + render: syncLagTime => {toSeconds(syncLagTime)} + }, + { + field: 'syncLagOps', + name: 'Sync Lag (ops)', + render: (syncLagOps, data) => ( + + {syncLagOps} +    + + Leader lag: {data.syncLagOpsLeader} +
+ Follower lag: {data.syncLagOpsFollower} + + )} + position="right" + /> + + ) + }, + { + field: 'error', + name: 'Error', + render: error => ( + + {error} + + ) + } + ]} + sorting={true} + pagination={pagination} + /> + ); + } + this.setState({ itemIdToExpandedRowMap }); + } + + renderTable() { + const { data } = this.props; + const items = data; + + let pagination = { + initialPageSize: 5, + pageSizeOptions: [5, 10, 20] + }; + + if (items.length <= pagination.initialPageSize) { + pagination = false; + } + + const sorting = { + sort: { + field: 'index', + direction: 'asc', + }, + }; + + return ( + { + const expanded = !!this.state.itemIdToExpandedRowMap[index]; + return ( + this.toggleShards(index, shards)}> + {index} +   + { expanded ? : } + + ); + } + }, + { + field: 'follows', + sortable: true, + name: 'Follows' + }, + { + field: 'opsSynced', + sortable: true, + name: 'Ops synced' + }, + { + field: 'syncLagTime', + sortable: true, + name: 'Last fetch time', + render: syncLagTime => {toSeconds(syncLagTime)} + }, + { + field: 'syncLagOps', + sortable: true, + name: 'Sync Lag (ops)', + }, + { + field: 'error', + sortable: true, + name: 'Error', + render: error => ( + + {error} + + ) + } + ]} + items={items} + pagination={pagination} + sorting={sorting} + itemId="id" + itemIdToExpandedRowMap={this.state.itemIdToExpandedRowMap} + /> + ); + } + + render() { + return ( + + + + + {this.renderTable()} + + + + + ); + } +} diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/ccr.test.js b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/ccr.test.js new file mode 100644 index 0000000000000..8df42974c6633 --- /dev/null +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/ccr.test.js @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { shallow } from 'enzyme'; +import { Ccr } from './ccr'; + +describe('Ccr', () => { + test('that it renders normally', () => { + const data = [ + { + follows: 'leader', + id: 'follower', + index: 'follower', + opsSynced: 400, + syncLagOps: 5, + syncLagTime: 60000, + shards: [ + { + opsSynced: 200, + shardId: 0, + syncLagOps: 2, + syncLagOpsFollower: 1, + syncLagOpsLeader: 1, + syncLagTime: 45000 + }, + { + opsSynced: 200, + shardId: 1, + syncLagOps: 1, + syncLagOpsFollower: 0, + syncLagOpsLeader: 1, + syncLagTime: 60000 + } + ] + }, + { + follows: 'leader2', + id: 'follower2', + index: 'follower2', + opsSynced: 50, + syncLagOps: 1, + syncLagTime: 12000, + error: 'not_working_properly', + shards: [ + { + opsSynced: 20, + shardId: 1, + syncLagOps: 0, + syncLagOpsFollower: 0, + syncLagOpsLeader: 0, + syncLagTime: 11000 + }, + { + opsSynced: 30, + shardId: 2, + syncLagOps: 5, + syncLagOpsFollower: 5, + syncLagOpsLeader: 0, + syncLagTime: 1000, + error: 'not_working_properly' + } + ] + } + ]; + + const component = shallow(); + expect(component).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/index.js b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/index.js new file mode 100644 index 0000000000000..11f150fadf754 --- /dev/null +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr/index.js @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { Ccr } from './ccr'; diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/__snapshots__/ccr_shard.test.js.snap b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/__snapshots__/ccr_shard.test.js.snap new file mode 100644 index 0000000000000..9a929395f6a0a --- /dev/null +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/__snapshots__/ccr_shard.test.js.snap @@ -0,0 +1,215 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`CcrShard that is renders an exception properly 1`] = ` + + +

+ + Errors + +

+ + + + +`; + +exports[`CcrShard that it renders normally 1`] = ` + + + + + + + + + + + + + + + + + + + + +

+ Advanced +

+ + } + id="ccrLatestStat" + initialIsOpen={false} + paddingSize="l" + > + + +

+ September 27, 2018 9:32:09 AM +

+ + + + { + "fetch_exceptions": [], + "follower_global_checkpoint": 3049, + "follower_index": "follower", + "follower_max_seq_no": 3049, + "last_requested_seq_no": 3049, + "leader_global_checkpoint": 3049, + "leader_index": "leader", + "leader_max_seq_no": 3049, + "mapping_version": 2, + "number_of_concurrent_reads": 1, + "number_of_concurrent_writes": 0, + "number_of_failed_bulk_operations": 0, + "number_of_failed_fetches": 0, + "number_of_operations_indexed": 3050, + "number_of_queued_writes": 0, + "number_of_successful_bulk_operations": 3050, + "number_of_successful_fetches": 3050, + "operations_received": 3050, + "shard_id": 0, + "time_since_last_fetch_millis": 9402, + "total_fetch_time_millis": 44128980, + "total_index_time_millis": 41827, + "total_transferred_bytes": 234156 +} + + + + + +`; diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/ccr_shard.js b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/ccr_shard.js new file mode 100644 index 0000000000000..8f34ca07320b9 --- /dev/null +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/ccr_shard.js @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, PureComponent } from 'react'; +import { + EuiPage, + EuiPageBody, + EuiPanel, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, + EuiTitle, + EuiBasicTable, + EuiCodeBlock, + EuiTextColor, + EuiHorizontalRule, + EuiAccordion, +} from '@elastic/eui'; +import { MonitoringTimeseriesContainer } from '../../chart'; +import { Status } from './status'; +import { formatDateTimeLocal } from '../../../../common/formatting'; + +export class CcrShard extends PureComponent { + renderCharts() { + const { metrics } = this.props; + const seriesToShow = [ + metrics.ccr_sync_lag_time, + metrics.ccr_sync_lag_ops + ]; + + const charts = seriesToShow.map((data, index) => ( + + + + + + )); + + return ( + + {charts} + + ); + } + + renderErrors() { + const { stat } = this.props; + if (stat.fetch_exceptions && stat.fetch_exceptions.length > 0) { + return ( + + + +

+ Errors +

+ + + + + + + ); + } + return null; + } + + renderLatestStat() { + const { stat, timestamp } = this.props; + + return ( +

Advanced

} + paddingSize="l" + > + + +

{formatDateTimeLocal(timestamp)}

+ + + + {JSON.stringify(stat, null, 2)} + + + + ); + } + + render() { + const { stat, oldestStat, formattedLeader } = this.props; + + return ( + + + + + {this.renderErrors()} + + {this.renderCharts()} + + + {this.renderLatestStat()} + + + ); + } +} diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/ccr_shard.test.js b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/ccr_shard.test.js new file mode 100644 index 0000000000000..41ca63aea4ee8 --- /dev/null +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/ccr_shard.test.js @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { shallow } from 'enzyme'; +import { CcrShard } from './ccr_shard'; + +describe('CcrShard', () => { + const props = { + formattedLeader: 'leader on remote', + metrics: [], + stat: { + fetch_exceptions: [], + follower_global_checkpoint: 3049, + follower_index: 'follower', + follower_max_seq_no: 3049, + last_requested_seq_no: 3049, + leader_global_checkpoint: 3049, + leader_index: 'leader', + leader_max_seq_no: 3049, + mapping_version: 2, + number_of_concurrent_reads: 1, + number_of_concurrent_writes: 0, + number_of_failed_bulk_operations: 0, + number_of_failed_fetches: 0, + number_of_operations_indexed: 3050, + number_of_queued_writes: 0, + number_of_successful_bulk_operations: 3050, + number_of_successful_fetches: 3050, + operations_received: 3050, + shard_id: 0, + time_since_last_fetch_millis: 9402, + total_fetch_time_millis: 44128980, + total_index_time_millis: 41827, + total_transferred_bytes: 234156, + }, + oldestStat: { + number_of_failed_fetches: 0, + number_of_operations_indexed: 2976 + }, + timestamp: '2018-09-27T13:32:09.412Z' + }; + + test('that it renders normally', () => { + const component = shallow(); + expect(component).toMatchSnapshot(); + }); + + test('that is renders an exception properly', () => { + const localProps = { + ...props, + stat: { + ...props.stat, + fetch_exceptions: [ + { + type: 'something_is_wrong', + reason: 'not sure but something happened' + } + ] + } + }; + + const component = shallow(); + expect(component.find('EuiPanel').get(0)).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/index.js b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/index.js new file mode 100644 index 0000000000000..98600c4163d30 --- /dev/null +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/index.js @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { CcrShard } from './ccr_shard'; diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/status.js b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/status.js new file mode 100644 index 0000000000000..1606e778f3a2a --- /dev/null +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/ccr_shard/status.js @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { SummaryStatus } from '../../summary_status'; +import { formatMetric } from '../../../lib/format_number'; + +export function Status({ stat, formattedLeader, oldestStat }) { + const { + follower_index: followerIndex, + shard_id: shardId, + number_of_operations_indexed: operationsReceived, + number_of_failed_fetches: failedFetches + } = stat; + + const { + number_of_operations_indexed: oldestOperationsReceived, + number_of_failed_fetches: oldestFailedFetches + } = oldestStat; + + const metrics = [ + { + label: 'Follower Index', + value: followerIndex, + dataTestSubj: 'followerIndex' + }, + { + label: 'Shard Id', + value: shardId, + dataTestSubj: 'shardId' + }, + { + label: 'Leader Index', + value: formattedLeader, + dataTestSubj: 'leaderIndex' + }, + { + label: 'Ops Synced', + value: formatMetric(operationsReceived - oldestOperationsReceived, 'int_commas'), + dataTestSubj: 'operationsReceived' + }, + { + label: 'Failed Fetches', + value: formatMetric(failedFetches - oldestFailedFetches, 'int_commas'), + dataTestSubj: 'failedFetches' + }, + ]; + + return ( + + ); +} diff --git a/x-pack/plugins/monitoring/public/directives/main/index.html b/x-pack/plugins/monitoring/public/directives/main/index.html index 96fee7dcbb9ce..9a1333a3dd57e 100644 --- a/x-pack/plugins/monitoring/public/directives/main/index.html +++ b/x-pack/plugins/monitoring/public/directives/main/index.html @@ -35,6 +35,7 @@ Nodes Indices Jobs + CCR response.data) + .catch((err) => { + const Private = $injector.get('Private'); + const ajaxErrorHandlers = Private(ajaxErrorHandlersProvider); + return ajaxErrorHandlers(err); + }); +} diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/index.html b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/index.html new file mode 100644 index 0000000000000..ca0b036ae39e1 --- /dev/null +++ b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/index.html @@ -0,0 +1,7 @@ + +
+ diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/index.js new file mode 100644 index 0000000000000..0c0f496af4676 --- /dev/null +++ b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/index.js @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import uiRoutes from 'ui/routes'; +import { getPageData } from './get_page_data'; +import template from './index.html'; +import { Ccr } from '../../../components/elasticsearch/ccr'; +import { MonitoringViewBaseController } from '../../base_controller'; + +uiRoutes.when('/elasticsearch/ccr', { + template, + resolve: { + pageData: getPageData, + }, + controllerAs: 'elasticsearchCcr', + controller: class ElasticsearchCcrController extends MonitoringViewBaseController { + constructor($injector, $scope) { + super({ + title: 'Elasticsearch - Ccr', + reactNodeId: 'elasticsearchCcrReact', + getPageData, + $scope, + $injector + }); + + $scope.$watch(() => this.data, data => { + this.renderReact(data); + }); + + this.renderReact = ({ data }) => { + super.renderReact( + + ); + }; + } + } +}); diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/get_page_data.js b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/get_page_data.js new file mode 100644 index 0000000000000..1b109995b89e1 --- /dev/null +++ b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/get_page_data.js @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ajaxErrorHandlersProvider } from 'plugins/monitoring/lib/ajax_error_handler'; +import { timefilter } from 'ui/timefilter'; + +export function getPageData($injector) { + const $http = $injector.get('$http'); + const $route = $injector.get('$route'); + const globalState = $injector.get('globalState'); + const timeBounds = timefilter.getBounds(); + const url = `../api/monitoring/v1/clusters/${globalState.cluster_uuid}/elasticsearch/ccr/${$route.current.params.index}/shard/${$route.current.params.shardId}`; // eslint-disable-line max-len + + return $http.post(url, { + ccs: globalState.ccs, + timeRange: { + min: timeBounds.min.toISOString(), + max: timeBounds.max.toISOString() + } + }) + .then(response => response.data) + .catch((err) => { + const Private = $injector.get('Private'); + const ajaxErrorHandlers = Private(ajaxErrorHandlersProvider); + return ajaxErrorHandlers(err); + }); + +} diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/index.html b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/index.html new file mode 100644 index 0000000000000..76469e5d9add5 --- /dev/null +++ b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/index.html @@ -0,0 +1,8 @@ + +
+ diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/index.js new file mode 100644 index 0000000000000..4242ef16405a0 --- /dev/null +++ b/x-pack/plugins/monitoring/public/views/elasticsearch/ccr/shard/index.js @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { get } from 'lodash'; +import uiRoutes from 'ui/routes'; +import { getPageData } from './get_page_data'; +import { routeInitProvider } from 'plugins/monitoring/lib/route_init'; +import template from './index.html'; +import { MonitoringViewBaseController } from '../../../base_controller'; +import { CcrShard } from '../../../../components/elasticsearch/ccr_shard'; + +uiRoutes.when('/elasticsearch/ccr/:index/shard/:shardId', { + template, + resolve: { + clusters: function (Private) { + const routeInit = Private(routeInitProvider); + return routeInit(); + }, + pageData: getPageData, + }, + controllerAs: 'elasticsearchCcr', + controller: class ElasticsearchCcrController extends MonitoringViewBaseController { + constructor($injector, $scope, pageData) { + super({ + title: 'Elasticsearch - Ccr - Shard', + reactNodeId: 'elasticsearchCcrShardReact', + getPageData, + $scope, + $injector + }); + + $scope.instance = `Index: ${get(pageData, 'stat.follower_index')} Shard: ${get(pageData, 'stat.shard_id')}`; + + $scope.$watch(() => this.data, data => { + this.renderReact(data); + }); + + this.renderReact = (props) => { + super.renderReact( + + ); + }; + } + } +}); diff --git a/x-pack/plugins/monitoring/server/lib/details/get_series.js b/x-pack/plugins/monitoring/server/lib/details/get_series.js index df6bad401f09c..306e93273c157 100644 --- a/x-pack/plugins/monitoring/server/lib/details/get_series.js +++ b/x-pack/plugins/monitoring/server/lib/details/get_series.js @@ -184,6 +184,7 @@ function handleSeries(metric, min, max, bucketSizeInSeconds, response) { const lastUsableBucketIndex = findLastUsableBucketIndex(buckets, max, firstUsableBucketIndex, bucketSizeInSeconds * 1000); let data = []; + if (firstUsableBucketIndex <= lastUsableBucketIndex) { // map buckets to values for charts const key = derivative ? 'metric_deriv.normalized_value' : 'metric.value'; diff --git a/x-pack/plugins/monitoring/server/lib/metrics/__test__/__snapshots__/metrics.test.js.snap b/x-pack/plugins/monitoring/server/lib/metrics/__test__/__snapshots__/metrics.test.js.snap index 26586c824fdec..a3590788367c3 100644 --- a/x-pack/plugins/monitoring/server/lib/metrics/__test__/__snapshots__/metrics.test.js.snap +++ b/x-pack/plugins/monitoring/server/lib/metrics/__test__/__snapshots__/metrics.test.js.snap @@ -1777,6 +1777,48 @@ Object { "units": "", "uuidField": "beats_stats.beat.uuid", }, + "ccr_sync_lag_ops": DifferenceMetric { + "aggs": Object { + "metric2_max": Object { + "max": Object { + "field": "ccr_stats.follower_global_checkpoint", + }, + }, + "metric_max": Object { + "max": Object { + "field": "ccr_stats.leader_max_seq_no", + }, + }, + }, + "app": "elasticsearch", + "calculation": [Function], + "derivative": false, + "description": "The number of operations the follower index is lagging behind the leader.", + "field": "", + "format": "0,0.[00]", + "label": "Ops delay", + "metricAgg": "sum", + "timestampField": "timestamp", + "title": "Ops delay", + "type": "ccr", + "units": "", + "uuidField": "source_node.uuid", + }, + "ccr_sync_lag_time": MillisecondsToSecondsMetric { + "app": "elasticsearch", + "calculation": [Function], + "derivative": false, + "description": "The amount of time the follower index is lagging behind the leader.", + "field": "ccr_stats.time_since_last_fetch_millis", + "format": "0.[00]", + "label": "Fetch delay", + "metricAgg": "max", + "timestampField": "timestamp", + "title": "Fetch delay", + "type": "ccr", + "units": "s", + "uuidField": "source_node.uuid", + }, "cluster_index_latency": LatencyMetric { "aggs": Object { "event_time_in_millis": Object { diff --git a/x-pack/plugins/monitoring/server/lib/metrics/elasticsearch/classes.js b/x-pack/plugins/monitoring/server/lib/metrics/elasticsearch/classes.js index b68cf3c948678..656465ba841dc 100644 --- a/x-pack/plugins/monitoring/server/lib/metrics/elasticsearch/classes.js +++ b/x-pack/plugins/monitoring/server/lib/metrics/elasticsearch/classes.js @@ -33,6 +33,35 @@ export class ElasticsearchMetric extends Metric { } } +export class DifferenceMetric extends ElasticsearchMetric { + constructor({ fieldSource, metric, metric2, ...opts }) { + super({ + ...opts, + field: '', // NOTE: this is not used for this + format: LARGE_FLOAT, + metricAgg: 'sum', // NOTE: this is used for a pointless aggregation + }); + + this.checkRequiredParams({ + metric, + metric2 + }); + + this.aggs = { + metric_max: { + max: { field: `${fieldSource}.${metric}` } + }, + metric2_max: { + max: { field: `${fieldSource}.${metric2}` } + }, + }; + + this.calculation = (bucket) => { + return _.get(bucket, 'metric_max.value') - _.get(bucket, 'metric2_max.value'); + }; + } +} + export class LatencyMetric extends ElasticsearchMetric { constructor({ metric, fieldSource, ...opts }) { super({ @@ -293,3 +322,16 @@ export class WriteThreadPoolRejectedMetric extends ElasticsearchMetric { }; } } + +export class MillisecondsToSecondsMetric extends ElasticsearchMetric { + constructor(opts) { + super({ + ...opts, + units: 's', + }); + + this.calculation = bucket => { + return _.get(bucket, 'metric.value') / 1000; + }; + } +} diff --git a/x-pack/plugins/monitoring/server/lib/metrics/elasticsearch/metrics.js b/x-pack/plugins/monitoring/server/lib/metrics/elasticsearch/metrics.js index 592b4147edd98..1a7639f3e097b 100644 --- a/x-pack/plugins/monitoring/server/lib/metrics/elasticsearch/metrics.js +++ b/x-pack/plugins/monitoring/server/lib/metrics/elasticsearch/metrics.js @@ -15,7 +15,9 @@ import { ThreadPoolQueueMetric, ThreadPoolRejectedMetric, WriteThreadPoolQueueMetric, - WriteThreadPoolRejectedMetric + WriteThreadPoolRejectedMetric, + DifferenceMetric, + MillisecondsToSecondsMetric, } from './classes'; import { LARGE_FLOAT, @@ -944,5 +946,29 @@ export const metrics = { units: '', type: 'index', derivative: true - }) + }), + + // CCR + ccr_sync_lag_time: new MillisecondsToSecondsMetric({ + title: 'Fetch delay', // title to use for the chart + type: 'ccr', + field: 'ccr_stats.time_since_last_fetch_millis', + label: 'Fetch delay', + description: 'The amount of time the follower index is lagging behind the leader.', + format: SMALL_FLOAT, + metricAgg: 'max', + units: 'ms' + }), + ccr_sync_lag_ops: new DifferenceMetric({ + title: 'Ops delay', // title to use for the chart + type: 'ccr', + fieldSource: 'ccr_stats', + metric: 'leader_max_seq_no', + metric2: 'follower_global_checkpoint', + label: 'Ops delay', + description: 'The number of operations the follower index is lagging behind the leader.', + format: SMALL_FLOAT, + metricAgg: 'max', + units: '' + }), }; diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.js b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.js new file mode 100644 index 0000000000000..c2481d71acee8 --- /dev/null +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr.js @@ -0,0 +1,258 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Joi from 'joi'; +import moment from 'moment'; +import { get, groupBy } from 'lodash'; +import { handleError } from '../../../../lib/errors/handle_error'; +import { prefixIndexPattern } from '../../../../lib/ccs_utils'; + +function getBucketScript(max, min) { + return { + bucket_script: { + buckets_path: { + max, + min, + }, + script: 'params.max - params.min' + } + }; +} + +function buildRequest(req, config, esIndexPattern) { + const min = moment.utc(req.payload.timeRange.min).valueOf(); + const max = moment.utc(req.payload.timeRange.max).valueOf(); + const maxBucketSize = config.get('xpack.monitoring.max_bucket_size'); + const aggs = { + ops_synced_max: { + max: { + field: 'ccr_stats.number_of_operations_indexed' + } + }, + ops_synced_min: { + min: { + field: 'ccr_stats.number_of_operations_indexed' + } + }, + + last_fetch_time_max: { + max: { + field: 'ccr_stats.time_since_last_fetch_millis' + } + }, + last_fetch_time_min: { + min: { + field: 'ccr_stats.time_since_last_fetch_millis' + } + }, + lag_ops_leader_max: { + max: { + field: 'ccr_stats.leader_max_seq_no' + } + }, + lag_ops_leader_min: { + min: { + field: 'ccr_stats.leader_max_seq_no' + } + }, + lag_ops_global_max: { + max: { + field: 'ccr_stats.follower_global_checkpoint' + } + }, + lag_ops_global_min: { + min: { + field: 'ccr_stats.follower_global_checkpoint' + } + }, + leader_lag_ops_checkpoint_max: { + max: { + field: 'ccr_stats.leader_global_checkpoint' + } + }, + leader_lag_ops_checkpoint_min: { + min: { + field: 'ccr_stats.leader_global_checkpoint' + } + }, + + last_fetch_time: getBucketScript('last_fetch_time_max', 'last_fetch_time_min'), + ops_synced: getBucketScript('ops_synced_max', 'ops_synced_min'), + lag_ops_leader: getBucketScript('lag_ops_leader_max', 'lag_ops_leader_min'), + lag_ops_global: getBucketScript('lag_ops_global_max', 'lag_ops_global_min'), + lag_ops: getBucketScript('lag_ops_leader', 'lag_ops_global'), + lag_ops_leader_checkpoint: getBucketScript('leader_lag_ops_checkpoint_max', 'leader_lag_ops_checkpoint_min'), + leader_lag_ops: getBucketScript('lag_ops_leader', 'lag_ops_leader_checkpoint'), + follower_lag_ops: getBucketScript('lag_ops_leader_checkpoint', 'lag_ops_global'), + }; + + return { + index: esIndexPattern, + size: maxBucketSize, + filterPath: [ + 'hits.hits.inner_hits.by_shard.hits.hits._source.ccr_stats.fetch_exceptions', + 'hits.hits.inner_hits.by_shard.hits.hits._source.ccr_stats.follower_index', + 'hits.hits.inner_hits.by_shard.hits.hits._source.ccr_stats.shard_id', + 'aggregations.by_follower_index.buckets.key', + 'aggregations.by_follower_index.buckets.leader_index.buckets.key', + 'aggregations.by_follower_index.buckets.by_shard_id.buckets.key', + 'aggregations.by_follower_index.buckets.by_shard_id.buckets.last_fetch_time.value', + 'aggregations.by_follower_index.buckets.by_shard_id.buckets.ops_synced.value', + 'aggregations.by_follower_index.buckets.by_shard_id.buckets.lag_ops.value', + 'aggregations.by_follower_index.buckets.by_shard_id.buckets.leader_lag_ops.value', + 'aggregations.by_follower_index.buckets.by_shard_id.buckets.follower_lag_ops.value', + ], + body: { + sort: [{ timestamp: { order: 'desc' } }], + query: { + bool: { + must: [ + { + term: { + type: { + value: 'ccr_stats' + } + } + }, + { + range: { + timestamp: { + format: 'epoch_millis', + gte: min, + lte: max, + } + } + } + ] + } + }, + collapse: { + field: 'ccr_stats.follower_index', + inner_hits: { + name: 'by_shard', + sort: [{ timestamp: 'desc' }], + size: maxBucketSize, + collapse: { + field: 'ccr_stats.shard_id', + } + } + }, + aggs: { + by_follower_index: { + terms: { + field: 'ccr_stats.follower_index', + size: maxBucketSize, + }, + aggs: { + leader_index: { + terms: { + field: 'ccr_stats.leader_index', + size: 1 + } + }, + by_shard_id: { + terms: { + field: 'ccr_stats.shard_id', + size: 10 + }, + aggs, + } + } + } + } + } + }; +} + +export function ccrRoute(server) { + server.route({ + method: 'POST', + path: '/api/monitoring/v1/clusters/{clusterUuid}/elasticsearch/ccr', + config: { + validate: { + params: Joi.object({ + clusterUuid: Joi.string().required() + }), + payload: Joi.object({ + ccs: Joi.string().optional(), + timeRange: Joi.object({ + min: Joi.date().required(), + max: Joi.date().required() + }).required() + }) + } + }, + async handler(req, reply) { + const config = server.config(); + const ccs = req.payload.ccs; + const esIndexPattern = prefixIndexPattern(config, 'xpack.monitoring.elasticsearch.index_pattern', ccs); + + try { + const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); + const response = await callWithRequest(req, 'search', buildRequest(req, config, esIndexPattern)); + + if (!response || Object.keys(response).length === 0) { + reply({ data: [] }); + return; + } + + const fullStats = get(response, 'hits.hits').reduce((accum, hit) => { + const innerHits = get(hit, 'inner_hits.by_shard.hits.hits'); + const innerHitsSource = innerHits.map(innerHit => get(innerHit, '_source.ccr_stats')); + const grouped = groupBy(innerHitsSource, stat => `${stat.follower_index}:${stat.shard_id}`); + + return { + ...accum, + ...grouped + }; + }, {}); + + const buckets = get(response, 'aggregations.by_follower_index.buckets'); + const data = buckets.reduce((accum, bucket) => { + const leaderIndex = get(bucket, 'leader_index.buckets[0].key'); + let follows = leaderIndex; + if (follows.includes(':')) { + const followsSplit = follows.split(':'); + follows = `${followsSplit[1]} on ${followsSplit[0]}`; + } + + const stat = { + id: bucket.key, + index: bucket.key, + follows, + }; + + stat.shards = get(bucket, 'by_shard_id.buckets').reduce((accum, shardBucket) => { + const fullStat = get(fullStats[`${bucket.key}:${shardBucket.key}`], '[0]', {}); + const shardStat = { + shardId: shardBucket.key, + error: fullStat.fetch_exceptions.length ? fullStat.fetch_exceptions[0].exception.type : null, + opsSynced: get(shardBucket, 'ops_synced.value'), + syncLagTime: get(shardBucket, 'last_fetch_time.value'), + syncLagOps: get(shardBucket, 'lag_ops.value'), + syncLagOpsLeader: get(shardBucket, 'leader_lag_ops.value'), + syncLagOpsFollower: get(shardBucket, 'follower_lag_ops.value'), + }; + accum.push(shardStat); + return accum; + }, []); + + stat.error = (stat.shards.find(shard => shard.error) || {}).error; + stat.opsSynced = stat.shards.reduce((sum, { opsSynced }) => sum + opsSynced, 0); + stat.syncLagTime = stat.shards.reduce((max, { syncLagTime }) => Math.max(max, syncLagTime), 0); + stat.syncLagOps = stat.shards.reduce((max, { syncLagOps }) => Math.max(max, syncLagOps), 0); + + accum.push(stat); + return accum; + }, []); + + reply({ data }); + } catch(err) { + reply(handleError(err, req)); + } + } + }); +} diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr_shard.js b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr_shard.js new file mode 100644 index 0000000000000..87b4d91515ea4 --- /dev/null +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/ccr_shard.js @@ -0,0 +1,149 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; +import moment from 'moment'; +import Joi from 'joi'; +import { handleError } from '../../../../lib/errors/handle_error'; +import { prefixIndexPattern } from '../../../../lib/ccs_utils'; +import { getMetrics } from '../../../../lib/details/get_metrics'; + +function getFormattedLeaderIndex(leaderIndex) { + let leader = leaderIndex; + if (leader.includes(':')) { + const leaderSplit = leader.split(':'); + leader = `${leaderSplit[1]} on ${leaderSplit[0]}`; + } + return leader; +} + +async function getCcrStat(req, esIndexPattern, filters) { + const min = moment.utc(req.payload.timeRange.min).valueOf(); + const max = moment.utc(req.payload.timeRange.max).valueOf(); + + const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring'); + + const params = { + index: esIndexPattern, + size: 1, + filterPath: [ + 'hits.hits._source.ccr_stats', + 'hits.hits._source.timestamp', + 'hits.hits.inner_hits.oldest.hits.hits._source.ccr_stats.number_of_operations_indexed', + 'hits.hits.inner_hits.oldest.hits.hits._source.ccr_stats.number_of_failed_fetches', + ], + body: { + sort: [{ timestamp: { order: 'desc' } }], + query: { + bool: { + must: [ + ...filters, + { + range: { + timestamp: { + format: 'epoch_millis', + gte: min, + lte: max, + } + } + } + ] + } + }, + collapse: { + field: 'ccr_stats.follower_index', + inner_hits: { + name: 'oldest', + size: 1, + sort: [{ timestamp: 'asc' }] + } + } + } + }; + + return await callWithRequest(req, 'search', params); +} + +export function ccrShardRoute(server) { + server.route({ + method: 'POST', + path: '/api/monitoring/v1/clusters/{clusterUuid}/elasticsearch/ccr/{index}/shard/{shardId}', + config: { + validate: { + params: Joi.object({ + clusterUuid: Joi.string().required(), + index: Joi.string().required(), + shardId: Joi.string().required() + }), + payload: Joi.object({ + ccs: Joi.string().optional(), + timeRange: Joi.object({ + min: Joi.date().required(), + max: Joi.date().required() + }).required(), + }) + } + }, + async handler(req, reply) { + const config = server.config(); + const index = req.params.index; + const shardId = req.params.shardId; + const ccs = req.payload.ccs; + const esIndexPattern = prefixIndexPattern(config, 'xpack.monitoring.elasticsearch.index_pattern', ccs); + + const filters = [ + { + term: { + type: { + value: 'ccr_stats' + } + } + }, + { + term: { + 'ccr_stats.follower_index': { + value: index, + } + } + }, + { + term: { + 'ccr_stats.shard_id': { + value: shardId, + } + } + } + ]; + + try { + + const [ + metrics, + ccrResponse + ] = await Promise.all([ + getMetrics(req, esIndexPattern, [ + { keys: ['ccr_sync_lag_time'], name: 'ccr_sync_lag_time' }, + { keys: ['ccr_sync_lag_ops'], name: 'ccr_sync_lag_ops' }, + ], filters), + getCcrStat(req, esIndexPattern, filters) + ]); + + const stat = get(ccrResponse, 'hits.hits[0]._source.ccr_stats', {}); + const oldestStat = get(ccrResponse, 'hits.hits[0].inner_hits.oldest.hits.hits[0]._source.ccr_stats', {}); + + reply({ + metrics, + stat, + formattedLeader: getFormattedLeaderIndex(stat.leader_index), + timestamp: get(ccrResponse, 'hits.hits[0]._source.timestamp'), + oldestStat, + }); + } catch(err) { + reply(handleError(err, req)); + } + } + }); +} diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/index.js b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/index.js index aeab4d3832102..b4697f1de51c3 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/index.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/elasticsearch/index.js @@ -10,3 +10,5 @@ export { esNodeRoute } from './node_detail'; export { esNodesRoute } from './nodes'; export { esOverviewRoute } from './overview'; export { mlJobRoute } from './ml_jobs'; +export { ccrRoute } from './ccr'; +export { ccrShardRoute } from './ccr_shard'; diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/ui.js b/x-pack/plugins/monitoring/server/routes/api/v1/ui.js index b62406841955c..c07a35109643d 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/ui.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/ui.js @@ -26,7 +26,9 @@ export { esNodeRoute, esNodesRoute, esOverviewRoute, - mlJobRoute + mlJobRoute, + ccrRoute, + ccrShardRoute } from './elasticsearch'; export { clusterSettingsCheckRoute, diff --git a/x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr.js b/x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr.js new file mode 100644 index 0000000000000..249baa8c27915 --- /dev/null +++ b/x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr.js @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from 'expect.js'; +import ccrFixture from './fixtures/ccr'; + +export default function ({ getService }) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + describe('ccr', () => { + const archive = 'monitoring/ccr'; + const timeRange = { + min: '2018-09-19T00:00:00.000Z', + max: '2018-09-19T23:59:59.000Z' + }; + + before('load archive', () => { + return esArchiver.load(archive); + }); + + after('unload archive', () => { + return esArchiver.unload(archive); + }); + + it('should return all followers and a grouping of stats by follower index', async () => { + const { body } = await supertest + .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr') + .set('kbn-xsrf', 'xxx') + .send({ + timeRange, + }) + .expect(200); + + expect(body).to.eql(ccrFixture); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_shard.js b/x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_shard.js new file mode 100644 index 0000000000000..e5a2418f215b9 --- /dev/null +++ b/x-pack/test/api_integration/apis/monitoring/elasticsearch/ccr_shard.js @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from 'expect.js'; +import ccrShardFixture from './fixtures/ccr_shard'; + +export default function ({ getService }) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + describe('ccr shard', () => { + const archive = 'monitoring/ccr'; + const timeRange = { + min: '2018-09-19T00:00:00.000Z', + max: '2018-09-19T23:59:59.000Z' + }; + + before('load archive', () => { + return esArchiver.load(archive); + }); + + after('unload archive', () => { + return esArchiver.unload(archive); + }); + + it('should return specific shard details', async () => { + const { body } = await supertest + .post('/api/monitoring/v1/clusters/YCxj-RAgSZCP6GuOQ8M1EQ/elasticsearch/ccr/follower/shard/0') + .set('kbn-xsrf', 'xxx') + .send({ + timeRange, + }) + .expect(200); + + expect(body).to.eql(ccrShardFixture); + }); + }); +} diff --git a/x-pack/test/api_integration/apis/monitoring/elasticsearch/fixtures/ccr.json b/x-pack/test/api_integration/apis/monitoring/elasticsearch/fixtures/ccr.json new file mode 100644 index 0000000000000..6ac135c8bc858 --- /dev/null +++ b/x-pack/test/api_integration/apis/monitoring/elasticsearch/fixtures/ccr.json @@ -0,0 +1,83 @@ +{ + "data": [{ + "id": "follower2", + "index": "follower2", + "follows": "leader2", + "shards": [{ + "shardId": 0, + "error": null, + "opsSynced": 52, + "syncLagTime": 59881, + "syncLagOps": 0, + "syncLagOpsLeader": 0, + "syncLagOpsFollower": 0 + }, { + "shardId": 1, + "error": null, + "opsSynced": 47, + "syncLagTime": 59959, + "syncLagOps": 0, + "syncLagOpsLeader": 0, + "syncLagOpsFollower": 0 + }, { + "shardId": 2, + "error": null, + "opsSynced": 51, + "syncLagTime": 55229, + "syncLagOps": 0, + "syncLagOpsLeader": 0, + "syncLagOpsFollower": 0 + }, { + "shardId": 3, + "error": null, + "opsSynced": 50, + "syncLagTime": 50483, + "syncLagOps": 0, + "syncLagOpsLeader": 0, + "syncLagOpsFollower": 0 + }, { + "shardId": 4, + "error": null, + "opsSynced": 55, + "syncLagTime": 55554, + "syncLagOps": 0, + "syncLagOpsLeader": 0, + "syncLagOpsFollower": 0 + }], + "opsSynced": 255, + "syncLagTime": 59959, + "syncLagOps": 0 + }, { + "id": "follower", + "index": "follower", + "follows": "leader", + "shards": [{ + "shardId": 0, + "error": null, + "opsSynced": 85, + "syncLagTime": 45513, + "syncLagOps": 0, + "syncLagOpsLeader": 0, + "syncLagOpsFollower": 0 + }, { + "shardId": 1, + "error": null, + "opsSynced": 94, + "syncLagTime": 55205, + "syncLagOps": 0, + "syncLagOpsLeader": 0, + "syncLagOpsFollower": 0 + }, { + "shardId": 2, + "error": null, + "opsSynced": 76, + "syncLagTime": 50003, + "syncLagOps": 0, + "syncLagOpsLeader": 0, + "syncLagOpsFollower": 0 + }], + "opsSynced": 255, + "syncLagTime": 55205, + "syncLagOps": 0 + }] +} diff --git a/x-pack/test/api_integration/apis/monitoring/elasticsearch/fixtures/ccr_shard.json b/x-pack/test/api_integration/apis/monitoring/elasticsearch/fixtures/ccr_shard.json new file mode 100644 index 0000000000000..e0f0f8d8ff945 --- /dev/null +++ b/x-pack/test/api_integration/apis/monitoring/elasticsearch/fixtures/ccr_shard.json @@ -0,0 +1,75 @@ +{ + "formattedLeader": "leader", + "metrics": { + "ccr_sync_lag_time": [{ + "bucket_size": "10 min", + "timeRange": { + "min": 1537315200000, + "max": 1537401599000 + }, + "metric": { + "app": "elasticsearch", + "field": "ccr_stats.time_since_last_fetch_millis", + "metricAgg": "max", + "label": "Fetch delay", + "title": "Fetch delay", + "description": "The amount of time the follower index is lagging behind the leader.", + "units": "s", + "format": "0.[00]", + "hasCalculation": true, + "isDerivative": false + }, + "data": [] + }], + "ccr_sync_lag_ops": [{ + "bucket_size": "10 min", + "timeRange": { + "min": 1537315200000, + "max": 1537401599000 + }, + "metric": { + "app": "elasticsearch", + "field": "", + "metricAgg": "sum", + "label": "Ops delay", + "title": "Ops delay", + "description": "The amount of time the follower index is lagging behind the leader.", + "units": "ms", + "format": "0,0.[00]", + "hasCalculation": true, + "isDerivative": false + }, + "data": [] + }] + }, + "stat": { + "leader_index": "leader", + "follower_index": "follower", + "shard_id": 0, + "leader_global_checkpoint": 85, + "leader_max_seq_no": 85, + "follower_global_checkpoint": 85, + "follower_max_seq_no": 85, + "last_requested_seq_no": 85, + "number_of_concurrent_reads": 1, + "number_of_concurrent_writes": 0, + "number_of_queued_writes": 0, + "mapping_version": 2, + "total_fetch_time_millis": 1265908, + "number_of_successful_fetches": 86, + "number_of_failed_fetches": 0, + "operations_received": 86, + "total_transferred_bytes": 6602, + "total_index_time_millis": 1096, + "number_of_successful_bulk_operations": 86, + "number_of_failed_bulk_operations": 0, + "number_of_operations_indexed": 86, + "fetch_exceptions": [], + "time_since_last_fetch_millis": 19886 + }, + "timestamp": "2018-09-19T20:01:00.440Z", + "oldestStat": { + "number_of_failed_fetches": 0, + "number_of_operations_indexed": 1 + } +} diff --git a/x-pack/test/api_integration/apis/monitoring/elasticsearch/index.js b/x-pack/test/api_integration/apis/monitoring/elasticsearch/index.js index 1d3a0f7f5a587..6253c2542ce0e 100644 --- a/x-pack/test/api_integration/apis/monitoring/elasticsearch/index.js +++ b/x-pack/test/api_integration/apis/monitoring/elasticsearch/index.js @@ -12,5 +12,7 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./node_detail_advanced')); loadTestFile(require.resolve('./indices')); loadTestFile(require.resolve('./index_detail')); + loadTestFile(require.resolve('./ccr')); + loadTestFile(require.resolve('./ccr_shard')); }); } diff --git a/x-pack/test/functional/es_archives/monitoring/ccr/data.json.gz b/x-pack/test/functional/es_archives/monitoring/ccr/data.json.gz new file mode 100644 index 0000000000000..2ff098caffa56 Binary files /dev/null and b/x-pack/test/functional/es_archives/monitoring/ccr/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/monitoring/ccr/mappings.json b/x-pack/test/functional/es_archives/monitoring/ccr/mappings.json new file mode 100644 index 0000000000000..45542a21f87a9 --- /dev/null +++ b/x-pack/test/functional/es_archives/monitoring/ccr/mappings.json @@ -0,0 +1,1019 @@ +{ + "type": "index", + "value": { + "index": ".monitoring-es-6-2018.09.19", + "settings": { + "index": { + "codec": "best_compression", + "number_of_shards": "1", + "auto_expand_replicas": "0-1", + "format": "6", + "number_of_replicas": "0" + } + }, + "mappings": { + "doc": { + "dynamic": "false", + "date_detection": false, + "properties": { + "ccr_stats": { + "properties": { + "fetch_exceptions": { + "type": "nested", + "properties": { + "exception": { + "properties": { + "reason": { + "type": "text" + }, + "type": { + "type": "keyword" + } + } + }, + "from_seq_no": { + "type": "long" + }, + "retries": { + "type": "integer" + } + } + }, + "follower_global_checkpoint": { + "type": "long" + }, + "follower_index": { + "type": "keyword" + }, + "follower_max_seq_no": { + "type": "long" + }, + "last_requested_seq_no": { + "type": "long" + }, + "leader_global_checkpoint": { + "type": "long" + }, + "leader_index": { + "type": "keyword" + }, + "leader_max_seq_no": { + "type": "long" + }, + "mapping_version": { + "type": "long" + }, + "number_of_concurrent_reads": { + "type": "long" + }, + "number_of_concurrent_writes": { + "type": "long" + }, + "number_of_failed_bulk_operations": { + "type": "long" + }, + "number_of_failed_fetches": { + "type": "long" + }, + "number_of_operations_indexed": { + "type": "long" + }, + "number_of_queued_writes": { + "type": "long" + }, + "number_of_successful_bulk_operations": { + "type": "long" + }, + "number_of_successful_fetches": { + "type": "long" + }, + "operations_received": { + "type": "long" + }, + "shard_id": { + "type": "integer" + }, + "time_since_last_fetch_millis": { + "type": "long" + }, + "total_fetch_time_millis": { + "type": "long" + }, + "total_index_time_millis": { + "type": "long" + }, + "total_transferred_bytes": { + "type": "long" + } + } + }, + "cluster_state": { + "properties": { + "master_node": { + "type": "keyword" + }, + "nodes": { + "type": "object" + }, + "nodes_hash": { + "type": "integer" + }, + "shards": { + "type": "object" + }, + "state_uuid": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "cluster_stats": { + "properties": { + "indices": { + "type": "object" + }, + "nodes": { + "type": "object" + } + } + }, + "cluster_uuid": { + "type": "keyword" + }, + "index_recovery": { + "type": "object" + }, + "index_stats": { + "properties": { + "index": { + "type": "keyword" + }, + "primaries": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "fielddata": { + "properties": { + "evictions": { + "type": "long" + }, + "memory_size_in_bytes": { + "type": "long" + } + } + }, + "indexing": { + "properties": { + "index_time_in_millis": { + "type": "long" + }, + "index_total": { + "type": "long" + }, + "throttle_time_in_millis": { + "type": "long" + } + } + }, + "merges": { + "properties": { + "total_size_in_bytes": { + "type": "long" + } + } + }, + "query_cache": { + "properties": { + "evictions": { + "type": "long" + }, + "hit_count": { + "type": "long" + }, + "memory_size_in_bytes": { + "type": "long" + }, + "miss_count": { + "type": "long" + } + } + }, + "refresh": { + "properties": { + "total_time_in_millis": { + "type": "long" + } + } + }, + "request_cache": { + "properties": { + "evictions": { + "type": "long" + }, + "hit_count": { + "type": "long" + }, + "memory_size_in_bytes": { + "type": "long" + }, + "miss_count": { + "type": "long" + } + } + }, + "search": { + "properties": { + "query_time_in_millis": { + "type": "long" + }, + "query_total": { + "type": "long" + } + } + }, + "segments": { + "properties": { + "count": { + "type": "integer" + }, + "doc_values_memory_in_bytes": { + "type": "long" + }, + "fixed_bit_set_memory_in_bytes": { + "type": "long" + }, + "index_writer_memory_in_bytes": { + "type": "long" + }, + "memory_in_bytes": { + "type": "long" + }, + "norms_memory_in_bytes": { + "type": "long" + }, + "points_memory_in_bytes": { + "type": "long" + }, + "stored_fields_memory_in_bytes": { + "type": "long" + }, + "term_vectors_memory_in_bytes": { + "type": "long" + }, + "terms_memory_in_bytes": { + "type": "long" + }, + "version_map_memory_in_bytes": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "fielddata": { + "properties": { + "evictions": { + "type": "long" + }, + "memory_size_in_bytes": { + "type": "long" + } + } + }, + "indexing": { + "properties": { + "index_time_in_millis": { + "type": "long" + }, + "index_total": { + "type": "long" + }, + "throttle_time_in_millis": { + "type": "long" + } + } + }, + "merges": { + "properties": { + "total_size_in_bytes": { + "type": "long" + } + } + }, + "query_cache": { + "properties": { + "evictions": { + "type": "long" + }, + "hit_count": { + "type": "long" + }, + "memory_size_in_bytes": { + "type": "long" + }, + "miss_count": { + "type": "long" + } + } + }, + "refresh": { + "properties": { + "total_time_in_millis": { + "type": "long" + } + } + }, + "request_cache": { + "properties": { + "evictions": { + "type": "long" + }, + "hit_count": { + "type": "long" + }, + "memory_size_in_bytes": { + "type": "long" + }, + "miss_count": { + "type": "long" + } + } + }, + "search": { + "properties": { + "query_time_in_millis": { + "type": "long" + }, + "query_total": { + "type": "long" + } + } + }, + "segments": { + "properties": { + "count": { + "type": "integer" + }, + "doc_values_memory_in_bytes": { + "type": "long" + }, + "fixed_bit_set_memory_in_bytes": { + "type": "long" + }, + "index_writer_memory_in_bytes": { + "type": "long" + }, + "memory_in_bytes": { + "type": "long" + }, + "norms_memory_in_bytes": { + "type": "long" + }, + "points_memory_in_bytes": { + "type": "long" + }, + "stored_fields_memory_in_bytes": { + "type": "long" + }, + "term_vectors_memory_in_bytes": { + "type": "long" + }, + "terms_memory_in_bytes": { + "type": "long" + }, + "version_map_memory_in_bytes": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + } + } + } + } + } + } + }, + "indices_stats": { + "properties": { + "_all": { + "properties": { + "primaries": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "indexing": { + "properties": { + "index_time_in_millis": { + "type": "long" + }, + "index_total": { + "type": "long" + } + } + }, + "search": { + "properties": { + "query_time_in_millis": { + "type": "long" + }, + "query_total": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "indexing": { + "properties": { + "index_time_in_millis": { + "type": "long" + }, + "index_total": { + "type": "long" + } + } + }, + "search": { + "properties": { + "query_time_in_millis": { + "type": "long" + }, + "query_total": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "interval_ms": { + "type": "long" + }, + "job_stats": { + "properties": { + "data_counts": { + "properties": { + "bucket_count": { + "type": "long" + }, + "earliest_record_timestamp": { + "type": "date" + }, + "empty_bucket_count": { + "type": "long" + }, + "input_bytes": { + "type": "long" + }, + "latest_record_timestamp": { + "type": "date" + }, + "processed_record_count": { + "type": "long" + }, + "sparse_bucket_count": { + "type": "long" + } + } + }, + "job_id": { + "type": "keyword" + }, + "model_size_stats": { + "properties": { + "bucket_allocation_failures_count": { + "type": "long" + }, + "model_bytes": { + "type": "long" + } + } + }, + "node": { + "properties": { + "id": { + "type": "keyword" + } + } + }, + "state": { + "type": "keyword" + } + } + }, + "node_stats": { + "properties": { + "fs": { + "properties": { + "data": { + "properties": { + "spins": { + "type": "boolean" + } + } + }, + "io_stats": { + "properties": { + "total": { + "properties": { + "operations": { + "type": "long" + }, + "read_kilobytes": { + "type": "long" + }, + "read_operations": { + "type": "long" + }, + "write_kilobytes": { + "type": "long" + }, + "write_operations": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "available_in_bytes": { + "type": "long" + }, + "free_in_bytes": { + "type": "long" + }, + "total_in_bytes": { + "type": "long" + } + } + } + } + }, + "indices": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + }, + "fielddata": { + "properties": { + "evictions": { + "type": "long" + }, + "memory_size_in_bytes": { + "type": "long" + } + } + }, + "indexing": { + "properties": { + "index_time_in_millis": { + "type": "long" + }, + "index_total": { + "type": "long" + }, + "throttle_time_in_millis": { + "type": "long" + } + } + }, + "query_cache": { + "properties": { + "evictions": { + "type": "long" + }, + "hit_count": { + "type": "long" + }, + "memory_size_in_bytes": { + "type": "long" + }, + "miss_count": { + "type": "long" + } + } + }, + "request_cache": { + "properties": { + "evictions": { + "type": "long" + }, + "hit_count": { + "type": "long" + }, + "memory_size_in_bytes": { + "type": "long" + }, + "miss_count": { + "type": "long" + } + } + }, + "search": { + "properties": { + "query_time_in_millis": { + "type": "long" + }, + "query_total": { + "type": "long" + } + } + }, + "segments": { + "properties": { + "count": { + "type": "integer" + }, + "doc_values_memory_in_bytes": { + "type": "long" + }, + "fixed_bit_set_memory_in_bytes": { + "type": "long" + }, + "index_writer_memory_in_bytes": { + "type": "long" + }, + "memory_in_bytes": { + "type": "long" + }, + "norms_memory_in_bytes": { + "type": "long" + }, + "points_memory_in_bytes": { + "type": "long" + }, + "stored_fields_memory_in_bytes": { + "type": "long" + }, + "term_vectors_memory_in_bytes": { + "type": "long" + }, + "terms_memory_in_bytes": { + "type": "long" + }, + "version_map_memory_in_bytes": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + } + } + } + } + }, + "jvm": { + "properties": { + "gc": { + "properties": { + "collectors": { + "properties": { + "old": { + "properties": { + "collection_count": { + "type": "long" + }, + "collection_time_in_millis": { + "type": "long" + } + } + }, + "young": { + "properties": { + "collection_count": { + "type": "long" + }, + "collection_time_in_millis": { + "type": "long" + } + } + } + } + } + } + }, + "mem": { + "properties": { + "heap_max_in_bytes": { + "type": "long" + }, + "heap_used_in_bytes": { + "type": "long" + }, + "heap_used_percent": { + "type": "half_float" + } + } + } + } + }, + "mlockall": { + "type": "boolean" + }, + "node_id": { + "type": "keyword" + }, + "node_master": { + "type": "boolean" + }, + "os": { + "properties": { + "cgroup": { + "properties": { + "cpu": { + "properties": { + "cfs_quota_micros": { + "type": "long" + }, + "control_group": { + "type": "keyword" + }, + "stat": { + "properties": { + "number_of_elapsed_periods": { + "type": "long" + }, + "number_of_times_throttled": { + "type": "long" + }, + "time_throttled_nanos": { + "type": "long" + } + } + } + } + }, + "cpuacct": { + "properties": { + "control_group": { + "type": "keyword" + }, + "usage_nanos": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "control_group": { + "type": "keyword" + }, + "limit_in_bytes": { + "type": "keyword" + }, + "usage_in_bytes": { + "type": "keyword" + } + } + } + } + }, + "cpu": { + "properties": { + "load_average": { + "properties": { + "15m": { + "type": "half_float" + }, + "1m": { + "type": "half_float" + }, + "5m": { + "type": "half_float" + } + } + } + } + } + } + }, + "process": { + "properties": { + "cpu": { + "properties": { + "percent": { + "type": "half_float" + } + } + }, + "max_file_descriptors": { + "type": "long" + }, + "open_file_descriptors": { + "type": "long" + } + } + }, + "thread_pool": { + "properties": { + "bulk": { + "properties": { + "queue": { + "type": "integer" + }, + "rejected": { + "type": "long" + }, + "threads": { + "type": "integer" + } + } + }, + "generic": { + "properties": { + "queue": { + "type": "integer" + }, + "rejected": { + "type": "long" + }, + "threads": { + "type": "integer" + } + } + }, + "get": { + "properties": { + "queue": { + "type": "integer" + }, + "rejected": { + "type": "long" + }, + "threads": { + "type": "integer" + } + } + }, + "index": { + "properties": { + "queue": { + "type": "integer" + }, + "rejected": { + "type": "long" + }, + "threads": { + "type": "integer" + } + } + }, + "maanagement": { + "properties": { + "queue": { + "type": "integer" + }, + "rejected": { + "type": "long" + }, + "threads": { + "type": "integer" + } + } + }, + "search": { + "properties": { + "queue": { + "type": "integer" + }, + "rejected": { + "type": "long" + }, + "threads": { + "type": "integer" + } + } + }, + "watcher": { + "properties": { + "queue": { + "type": "integer" + }, + "rejected": { + "type": "long" + }, + "threads": { + "type": "integer" + } + } + } + } + } + } + }, + "shard": { + "properties": { + "index": { + "type": "keyword" + }, + "node": { + "type": "keyword" + }, + "primary": { + "type": "boolean" + }, + "relocating_node": { + "type": "keyword" + }, + "shard": { + "type": "long" + }, + "state": { + "type": "keyword" + } + } + }, + "source_node": { + "properties": { + "host": { + "type": "keyword" + }, + "ip": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "timestamp": { + "type": "date", + "format": "date_time" + }, + "transport_address": { + "type": "keyword" + }, + "uuid": { + "type": "keyword" + } + } + }, + "state_uuid": { + "type": "keyword" + }, + "timestamp": { + "type": "date", + "format": "date_time" + }, + "type": { + "type": "keyword" + } + } + } + }, + "aliases": {} + } +} \ No newline at end of file