From 919b207a3d5011f058f26533f2120971a25ccded Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Tue, 21 May 2024 17:07:46 +0200 Subject: [PATCH 01/11] Add logic to display degraded fields in DQ Flyout --- .../src/scenarios/degraded_logs.ts | 27 +++++ .../dataset_quality/common/api_types.ts | 10 ++ .../common/data_streams_stats/types.ts | 21 +++- .../dataset_quality/common/es_fields/index.ts | 1 + .../dataset_quality/common/translations.ts | 27 +++++ .../flyout/degraded_fields/columns.tsx | 51 ++++++++ .../degraded_fields/degraded_fields.tsx | 39 ++++++ .../flyout/degraded_fields/table.tsx | 112 ++++++++++++++++++ .../flyout/flyout_summary/flyout_summary.tsx | 5 + .../hooks/use_dataset_quality_flyout.tsx | 5 + .../data_stream_details_client.ts | 41 ++++++- .../services/data_stream_details/types.ts | 3 + .../src/notifications.ts | 9 ++ .../src/state_machine.ts | 63 +++++++++- .../dataset_quality_controller/src/types.ts | 7 ++ .../data_streams/get_degraded_fields/index.ts | 81 +++++++++++++ .../server/routes/data_streams/routes.ts | 32 +++++ .../observability/server/index.ts | 9 +- .../observability/server/utils/queries.ts | 4 + 19 files changed, 539 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/columns.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/degraded_fields.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx create mode 100644 x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts diff --git a/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts index cb5313ac70795..d0afe09e3224f 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts @@ -115,6 +115,32 @@ const scenario: Scenario = async (runOptions) => { .timestamp(timestamp); }; + const datasetSynth4Logs = (i: number, timestamp: number) => { + const index = Math.floor(Math.random() * 3); + const isMalformed = i % 10 === 0; + return log + .create() + .dataset('synth.3') + .message(MESSAGE_LOG_LEVELS[index].message as string) + .service(SERVICE_NAMES[index]) + .defaults({ + 'trace.id': generateShortId(), + 'agent.name': 'synth-agent', + 'orchestrator.cluster.name': CLUSTER[index].clusterName, + 'orchestrator.cluster.id': CLUSTER[index].clusterId, + 'orchestrator.resource.id': generateShortId(), + 'cloud.provider': CLOUD_PROVIDERS[Math.floor(Math.random() * 3)], + 'cloud.region': CLOUD_REGION[index], + 'cloud.availability_zone': isMalformed + ? MORE_THAN_1024_CHARS // "ignore_above": 1024 in mapping + : `${CLOUD_REGION[index]}a`, + 'cloud.project.id': generateShortId(), + 'cloud.instance.id': generateShortId(), + 'log.file.path': `/logs/${generateLongId()}/error.txt`, + }) + .timestamp(timestamp + 100); + }; + const logs = range .interval('1m') .rate(1) @@ -125,6 +151,7 @@ const scenario: Scenario = async (runOptions) => { datasetSynth1Logs(timestamp), datasetSynth2Logs(index, timestamp), datasetSynth3Logs(index, timestamp), + datasetSynth4Logs(index, timestamp), ]); }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index d35748f9a7407..e2a918de285f1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -78,6 +78,16 @@ export const degradedDocsRt = rt.type({ export type DegradedDocs = rt.TypeOf; +export const degradedFieldRt = rt.type({ + fieldName: rt.string, + count: rt.number, + last_occurrence: rt.union([rt.null, rt.number]), +}); + +export const getDataStreamDegradedFieldsResponseRt = rt.array(degradedFieldRt); + +export type DegradedField = rt.TypeOf; + export const dataStreamSettingsRt = rt.partial({ createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless }); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index 9905cef5f7ea5..9bb8255b08032 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -29,11 +29,30 @@ export type GetDataStreamsDegradedDocsStatsResponse = export type DataStreamDegradedDocsStatServiceResponse = DegradedDocsStatType[]; export type DegradedDocsStatType = GetDataStreamsDegradedDocsStatsResponse['degradedDocs'][0]; +/* +Types for Degraded Fields inside a DataStream +*/ + +export type GetDataStreamDegradedFieldsPathParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/degraded_fields`>['params']['path']; +export type GetDataStreamDegradedFieldsQueryParams = + APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/degraded_fields`>['params']['query']; +export type GetDataStreamDegradedFieldsParams = GetDataStreamDegradedFieldsPathParams & + GetDataStreamDegradedFieldsQueryParams; + +/* +Types for DataStream Settings +*/ + export type GetDataStreamSettingsParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/settings`>['params']['path']; export type GetDataStreamSettingsResponse = APIReturnType<`GET /internal/dataset_quality/data_streams/{dataStream}/settings`>; +/* +Types for DataStream Details +*/ + type GetDataStreamDetailsPathParams = APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/{dataStream}/details`>['params']['path']; type GetDataStreamDetailsQueryParams = @@ -55,4 +74,4 @@ export type GetIntegrationDashboardsResponse = export type DashboardType = GetIntegrationDashboardsResponse['dashboards'][0]; export type { DataStreamStat } from './data_stream_stat'; -export type { DataStreamDetails, DataStreamSettings } from '../api_types'; +export type { DataStreamDetails, DataStreamSettings, DegradedField } from '../api_types'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/es_fields/index.ts b/x-pack/plugins/observability_solution/dataset_quality/common/es_fields/index.ts index 187093b7cd7f6..1e67fb1c68f81 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/es_fields/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/es_fields/index.ts @@ -6,6 +6,7 @@ */ export const _IGNORED = '_ignored'; +export const TIMESTAMP = '@timestamp'; export const DATA_STREAM_DATASET = 'data_stream.dataset'; export const DATA_STREAM_NAMESPACE = 'data_stream.namespace'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index d8ea867c44e4c..4944e8a054a53 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -137,6 +137,33 @@ export const flyoutShowAllText = i18n.translate('xpack.datasetQuality.flyoutShow defaultMessage: 'Show all', }); +export const flyoutImprovementText = i18n.translate( + 'xpack.datasetQuality.flyoutDegradedFieldsSectionTitle', + { + defaultMessage: 'Degraded Fields', + } +); + +export const flyoutImprovementTooltip = i18n.translate( + 'xpack.datasetQuality.flyoutDegradedFieldsSectionTooltip', + { + defaultMessage: 'Some logical tooltip for Improvements', + } +); + +export const flyoutDegradedFieldsTableLoadingText = i18n.translate( + 'xpack.datasetQuality.flyoutDegradedFieldsTableLoadingText', + { + defaultMessage: 'Loading degraded fields', + } +); + +export const flyoutDegradedFieldsTableNoData = i18n.translate( + 'xpack.datasetQuality.flyoutDegradedFieldsTableNoData', + { + defaultMessage: 'No degraded fields found', + } +); /* Summary Panel */ diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/columns.tsx new file mode 100644 index 0000000000000..7cc96aa665966 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/columns.tsx @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiBasicTableColumn } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { DegradedField } from '../../../../common/api_types'; + +const fieldColumnName = i18n.translate('xpack.datasetQuality.flyout.degradedField.field', { + defaultMessage: 'Field', +}); + +const countColumnName = i18n.translate('xpack.datasetQuality.flyout.degradedField.count', { + defaultMessage: 'Count', +}); + +const lastOccurrenceColumnName = i18n.translate( + 'xpack.datasetQuality.flyout.degradedField.last_occurrence', + { + defaultMessage: 'Last Occurrence', + } +); + +export const getDegradedFieldsColumns = (): Array> => [ + { + name: fieldColumnName, + field: 'fieldName', + }, + { + name: countColumnName, + sortable: true, + field: 'count', + truncateText: true, + }, + { + name: lastOccurrenceColumnName, + sortable: true, + field: 'last_occurrence', + render: (lastOccurrence: number) => { + if (lastOccurrence) { + const date = new Date(lastOccurrence); + return date.toISOString(); + } + + return ''; + }, + }, +]; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/degraded_fields.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/degraded_fields.tsx new file mode 100644 index 0000000000000..9995573725a82 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/degraded_fields.tsx @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiPanel, EuiTitle, EuiToolTip } from '@elastic/eui'; +import { css } from '@emotion/react'; +import { flyoutImprovementText, flyoutImprovementTooltip } from '../../../../common/translations'; +import { DegradedFieldTable } from './table'; + +export function DegradedFields() { + return ( + + + + +
{flyoutImprovementText}
+ + + + + + + + + + + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx new file mode 100644 index 0000000000000..5d242738f25ab --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiBasicTable, EuiEmptyPrompt } from '@elastic/eui'; +import React, { useMemo, useState } from 'react'; +import { orderBy } from 'lodash'; +import { getDegradedFieldsColumns } from './columns'; +import { Direction, useDatasetQualityFlyout } from '../../../hooks'; +import { + flyoutDegradedFieldsTableLoadingText, + flyoutDegradedFieldsTableNoData, +} from '../../../../common/translations'; +import { DegradedField } from '../../../../common/api_types'; + +const DEFAULT_SORT_FIELD = 'count'; +const DEFAULT_SORT_DIRECTION = 'asc'; +const DEFAULT_ROWS_PER_PAGE = 10; + +interface TableOptions { + page: { + index: number; + size: number; + }; + sort: { + field: keyof DegradedField; + direction: 'asc' | 'desc'; + }; +} + +const DEFAULT_TABLE_OPTIONS: TableOptions = { + page: { + index: 0, + size: 0, + }, + sort: { + field: DEFAULT_SORT_FIELD, + direction: DEFAULT_SORT_DIRECTION, + }, +}; + +export const DegradedFieldTable = () => { + const { degradedFields, loadingState } = useDatasetQualityFlyout(); + const columns = getDegradedFieldsColumns(); + const [tableOptions, setTableOptions] = useState(DEFAULT_TABLE_OPTIONS); + + const onTableChange = (options: { + page: { index: number; size: number }; + sort?: { field: keyof DegradedField; direction: Direction }; + }) => { + setTableOptions({ + page: { + index: options.page.index, + size: options.page.size, + }, + sort: { + field: options.sort?.field ?? DEFAULT_SORT_FIELD, + direction: options.sort?.direction ?? DEFAULT_SORT_DIRECTION, + }, + }); + }; + + const pagination = useMemo( + () => ({ + pageIndex: tableOptions.page.index, + pageSize: DEFAULT_ROWS_PER_PAGE, + totalItemCount: degradedFields?.length ?? 0, + hidePerPageOptions: true, + }), + [degradedFields, tableOptions] + ); + + const renderedItems = useMemo(() => { + const sortedItems = orderBy( + degradedFields, + tableOptions.sort.field, + tableOptions.sort.direction + ); + return sortedItems.slice( + tableOptions.page.index * DEFAULT_ROWS_PER_PAGE, + (tableOptions.page.index + 1) * DEFAULT_ROWS_PER_PAGE + ); + }, [degradedFields, tableOptions]); + + return ( + {flyoutDegradedFieldsTableNoData}} + hasBorder={false} + titleSize="m" + /> + ) + } + /> + ); +}; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx index c3e38d5a9940a..4977f6b0d7732 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/flyout_summary/flyout_summary.tsx @@ -26,6 +26,7 @@ import { useDatasetQualityContext } from '../../dataset_quality/context'; import { FlyoutDataset, TimeRangeConfig } from '../../../state_machines/dataset_quality_controller'; import { FlyoutSummaryHeader } from './flyout_summary_header'; import { FlyoutSummaryKpis, FlyoutSummaryKpisLoading } from './flyout_summary_kpis'; +import { DegradedFields } from '../degraded_fields/degraded_fields'; const nonAggregatableWarningTitle = i18n.translate('xpack.datasetQuality.nonAggregatable.title', { defaultMessage: 'Your request may take longer to complete', @@ -173,6 +174,10 @@ export function FlyoutSummary({ lastReloadTime={lastReloadTime} onTimeRangeChange={handleTimeRangeChange} /> + + + + ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx index f653cb2631fc8..8e6ec7f2fd691 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx @@ -23,6 +23,7 @@ export const useDatasetQualityFlyout = () => { insightsTimeRange, breakdownField, isNonAggregatable, + degradedFields, } = useSelector(service, (state) => state.context.flyout) ?? {}; const { timeRange } = useSelector(service, (state) => state.context.filters); @@ -31,6 +32,9 @@ export const useDatasetQualityFlyout = () => { dataStreamDetailsLoading: state.matches('flyout.initializing.dataStreamDetails.fetching'), dataStreamSettingsLoading: state.matches('flyout.initializing.dataStreamSettings.fetching'), datasetIntegrationsLoading: state.matches('flyout.initializing.integrationDashboards.fetching'), + datasetDegradedFieldsLoading: state.matches( + 'flyout.initializing.dataStreamDegradedFields.fetching' + ), })); return { @@ -43,5 +47,6 @@ export const useDatasetQualityFlyout = () => { breakdownField, loadingState, flyoutLoading: !dataStreamStat, + degradedFields, }; }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 3b95814397f3a..33a05ef5fe57e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -8,20 +8,24 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; import { - getDataStreamsSettingsResponseRt, + DegradedField, + getDataStreamDegradedFieldsResponseRt, getDataStreamsDetailsResponseRt, + getDataStreamsSettingsResponseRt, integrationDashboardsRT, } from '../../../common/api_types'; import { - GetDataStreamsStatsError, - GetDataStreamSettingsParams, - GetDataStreamSettingsResponse, + DataStreamDetails, + DataStreamSettings, + GetDataStreamDegradedFieldsParams, GetDataStreamDetailsParams, GetDataStreamDetailsResponse, + GetDataStreamSettingsParams, + GetDataStreamSettingsResponse, + GetDataStreamsStatsError, GetIntegrationDashboardsParams, GetIntegrationDashboardsResponse, } from '../../../common/data_streams_stats'; -import { DataStreamDetails, DataStreamSettings } from '../../../common/data_streams_stats'; import { IDataStreamDetailsClient } from './types'; export class DataStreamDetailsClient implements IDataStreamDetailsClient { @@ -66,6 +70,33 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { return dataStreamDetails as DataStreamDetails; } + public async getDataStreamDegradedFields({ + dataStream, + start, + end, + }: GetDataStreamDegradedFieldsParams) { + const response = await this.http + .get( + `/internal/dataset_quality/data_streams/${dataStream}/degraded_fields`, + { + query: { start, end }, + } + ) + .catch((error) => { + throw new GetDataStreamsStatsError( + `Failed to fetch data stream degraded fields": ${error}` + ); + }); + + return decodeOrThrow( + getDataStreamDegradedFieldsResponseRt, + (message: string) => + new GetDataStreamsStatsError( + `Failed to decode data stream degraded fields response: ${message}"` + ) + )(response); + } + public async getIntegrationDashboards({ integration }: GetIntegrationDashboardsParams) { const response = await this.http .get( diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts index f85e05d0179f5..807b79a1d87fe 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts @@ -13,6 +13,8 @@ import { DataStreamDetails, GetIntegrationDashboardsParams, GetIntegrationDashboardsResponse, + GetDataStreamDegradedFieldsParams, + DegradedField, } from '../../../common/data_streams_stats'; export type DataStreamDetailsServiceSetup = void; @@ -28,6 +30,7 @@ export interface DataStreamDetailsServiceStartDeps { export interface IDataStreamDetailsClient { getDataStreamSettings(params: GetDataStreamSettingsParams): Promise; getDataStreamDetails(params: GetDataStreamDetailsParams): Promise; + getDataStreamDegradedFields(params: GetDataStreamDegradedFieldsParams): Promise; getIntegrationDashboards( params: GetIntegrationDashboardsParams ): Promise; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts index 70da9d3d74e70..bf77eaa722ad3 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts @@ -44,6 +44,15 @@ export const fetchDegradedStatsFailedNotifier = (toasts: IToasts, error: Error) }); }; +export const fetchDegradedFieldsFailedNotifier = (toasts: IToasts, error: Error) => { + toasts.addDanger({ + title: i18n.translate('xpack.datasetQuality.fetchDegradedFieldsFailed', { + defaultMessage: "We couldn't get your degraded fields information.", + }), + text: error.message, + }); +}; + export const fetchNonAggregatableDatasetsFailedNotifier = (toasts: IToasts, error: Error) => { toasts.addDanger({ title: i18n.translate('xpack.datasetQuality.fetchNonAggregatableDatasetsFailed', { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index 8d8efc3d3c3db..91fbf67374296 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -8,7 +8,7 @@ import { IToasts } from '@kbn/core/public'; import { getDateISORange } from '@kbn/timerange'; import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate'; -import { DataStreamStat } from '../../../../common/api_types'; +import { DataStreamStat, DegradedField } from '../../../../common/api_types'; import { Integration } from '../../../../common/data_streams_stats/integration'; import { IDataStreamDetailsClient } from '../../../services/data_stream_details'; import { @@ -35,6 +35,7 @@ import { fetchIntegrationsFailedNotifier, noDatasetSelected, fetchNonAggregatableDatasetsFailedNotifier, + fetchDegradedFieldsFailedNotifier, } from './notifications'; import { DatasetQualityControllerContext, @@ -311,6 +312,31 @@ export const createPureDatasetQualityControllerStateMachine = ( }, }, }, + dataStreamDegradedFields: { + initial: 'fetching', + states: { + fetching: { + invoke: { + src: 'loadDegradedFieldsPerDataStream', + onDone: { + target: 'done', + actions: ['storeDegradedFields'], + }, + onError: { + target: 'done', + actions: ['notifyFetchDegradedFieldsFailed'], + }, + }, + }, + done: { + on: { + UPDATE_INSIGHTS_TIME_RANGE: { + target: 'fetching', + }, + }, + }, + }, + }, }, onDone: { target: '#DatasetQualityController.flyout.loaded', @@ -470,6 +496,16 @@ export const createPureDatasetQualityControllerStateMachine = ( } : {}; }), + storeDegradedFields: assign((context, event) => { + return 'data' in event + ? { + flyout: { + ...context.flyout, + degradedFields: (event.data ?? {}) as DegradedField[], + }, + } + : {}; + }), storeNonAggregatableDatasets: assign( ( _context: DefaultDatasetQualityControllerState, @@ -579,6 +615,8 @@ export const createDatasetQualityControllerStateMachine = ({ fetchDatasetStatsFailedNotifier(toasts, event.data), notifyFetchDegradedStatsFailed: (_context, event: DoneInvokeEvent) => fetchDegradedStatsFailedNotifier(toasts, event.data), + notifyFetchDegradedFieldsFailed: (_context, event: DoneInvokeEvent) => + fetchDegradedFieldsFailedNotifier(toasts, event.data), notifyFetchNonAggregatableDatasetsFailed: (_context, event: DoneInvokeEvent) => fetchNonAggregatableDatasetsFailedNotifier(toasts, event.data), notifyFetchDatasetSettingsFailed: (_context, event: DoneInvokeEvent) => @@ -606,6 +644,29 @@ export const createDatasetQualityControllerStateMachine = ({ end, }); }, + + loadDegradedFieldsPerDataStream: (context) => { + if (!context.flyout.dataset || !context.flyout.insightsTimeRange) { + fetchDatasetSettingsFailedNotifier(toasts, new Error(noDatasetSelected)); + + return Promise.resolve({}); + } + + const { startDate: start, endDate: end } = getDateISORange( + context.flyout.insightsTimeRange + ); + const { type, name: dataset, namespace } = context.flyout.dataset; + + return dataStreamDetailsClient.getDataStreamDegradedFields({ + dataStream: dataStreamPartsToIndexName({ + type: type as DataStreamType, + dataset, + namespace, + }), + start, + end, + }); + }, loadIntegrations: (context) => { return dataStreamStatsClient.getIntegrations({ type: context.type as GetIntegrationsParams['query']['type'], diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index 538ad16f3f977..1096f38181262 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -21,6 +21,7 @@ import { DataStreamStat, DataStreamStatType, GetNonAggregatableDataStreamsResponse, + DegradedField, } from '../../../../common/data_streams_stats'; export type FlyoutDataset = Omit< @@ -62,6 +63,7 @@ export interface WithFlyoutOptions { datasetDetails?: DataStreamDetails; insightsTimeRange?: TimeRangeConfig; breakdownField?: string; + degradedFields?: DegradedField[]; isNonAggregatable?: boolean; }; } @@ -140,6 +142,10 @@ export type DatasetQualityControllerTypeState = value: 'flyout.initializing.dataStreamDetails.fetching'; context: DefaultDatasetQualityStateContext; } + | { + value: 'flyout.initializing.dataStreamDegradedFields.fetching'; + context: DefaultDatasetQualityStateContext; + } | { value: 'flyout.initializing.integrationDashboards.fetching'; context: DefaultDatasetQualityStateContext; @@ -204,6 +210,7 @@ export type DatasetQualityControllerEvent = | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent + | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts new file mode 100644 index 0000000000000..2d45769fc23b2 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { rangeQuery, termQuery, existsQuery } from '@kbn/observability-plugin/server'; +import { DataStreamType } from '../../../../common/types'; +import { DegradedField } from '../../../../common/api_types'; +import { DEFAULT_DATASET_TYPE } from '../../../../common/constants'; +import { createDatasetQualityESClient } from '../../../utils'; +import { + _IGNORED, + DATA_STREAM_DATASET, + DATA_STREAM_NAMESPACE, + DATA_STREAM_TYPE, + TIMESTAMP, +} from '../../../../common/es_fields'; + +export async function getDegradedFields({ + esClient, + start, + end, + type = DEFAULT_DATASET_TYPE, + namespace, + dataset, +}: { + esClient: ElasticsearchClient; + start: number; + end: number; + type?: DataStreamType; + namespace: string; + dataset: string; +}): Promise { + const datasetQualityESClient = createDatasetQualityESClient(esClient); + + const filterQuery = [ + ...rangeQuery(start, end), + ...termQuery(DATA_STREAM_TYPE, type), + ...termQuery(DATA_STREAM_NAMESPACE, namespace), + ...termQuery(DATA_STREAM_DATASET, dataset), + ]; + + const mustQuery = [...existsQuery(_IGNORED)]; + + const aggs = { + degradedFields: { + terms: { + field: _IGNORED, + }, + aggs: { + last_occurrence: { + max: { + field: TIMESTAMP, + }, + }, + }, + }, + }; + + const response = await datasetQualityESClient.search({ + size: 0, + query: { + bool: { + filter: filterQuery, + must: mustQuery, + }, + }, + aggs, + }); + + return ( + response.aggregations?.degradedFields.buckets.map((bucket) => ({ + fieldName: bucket.key as string, + count: bucket.doc_count, + last_occurrence: bucket.last_occurrence.value, + })) ?? [] + ); +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index b4a757330647e..c4731483d8cec 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -13,6 +13,7 @@ import { DataStreamStat, DegradedDocs, NonAggregatableDatasets, + DegradedField, } from '../../../common/api_types'; import { indexNameToDataStreamParts } from '../../../common/utils'; import { rangeRt, typeRt } from '../../types/default_api_types'; @@ -22,6 +23,7 @@ import { getDataStreams } from './get_data_streams'; import { getDataStreamsStats } from './get_data_streams_stats'; import { getDegradedDocsPaginated } from './get_degraded_docs'; import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams'; +import { getDegradedFields } from './get_degraded_fields'; const statsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/stats', @@ -123,6 +125,35 @@ const nonAggregatableDatasetsRoute = createDatasetQualityServerRoute({ }, }); +const degradedFieldsRoute = createDatasetQualityServerRoute({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/degraded_fields', + params: t.type({ + path: t.type({ + dataStream: t.string, + }), + query: rangeRt, + }), + options: { + tags: [], + }, + async handler(resources): Promise { + const { context, params } = resources; + const { dataStream } = params.path; + const coreContext = await context.core; + + const esClient = coreContext.elasticsearch.client.asCurrentUser; + const { type, dataset, namespace } = indexNameToDataStreamParts(dataStream); + + return await getDegradedFields({ + esClient, + type, + dataset, + namespace, + ...params.query, + }); + }, +}); + const dataStreamSettingsRoute = createDatasetQualityServerRoute({ endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/settings', params: t.type({ @@ -198,6 +229,7 @@ export const dataStreamsRouteRepository = { ...statsRoute, ...degradedDocsRoute, ...nonAggregatableDatasetsRoute, + ...degradedFieldsRoute, ...dataStreamDetailsRoute, ...dataStreamSettingsRoute, }; diff --git a/x-pack/plugins/observability_solution/observability/server/index.ts b/x-pack/plugins/observability_solution/observability/server/index.ts index f7a9435198f8e..6486731fcd90c 100644 --- a/x-pack/plugins/observability_solution/observability/server/index.ts +++ b/x-pack/plugins/observability_solution/observability/server/index.ts @@ -20,7 +20,14 @@ import { WrappedElasticsearchClientError, } from '../common/utils/unwrap_es_response'; -export { rangeQuery, kqlQuery, termQuery, termsQuery, wildcardQuery } from './utils/queries'; +export { + rangeQuery, + kqlQuery, + termQuery, + termsQuery, + wildcardQuery, + existsQuery, +} from './utils/queries'; export { getParsedFilterQuery } from './utils/get_parsed_filtered_query'; export { getInspectResponse } from '../common/utils/get_inspect_response'; diff --git a/x-pack/plugins/observability_solution/observability/server/utils/queries.ts b/x-pack/plugins/observability_solution/observability/server/utils/queries.ts index fa581df62e745..e66fca3264f1b 100644 --- a/x-pack/plugins/observability_solution/observability/server/utils/queries.ts +++ b/x-pack/plugins/observability_solution/observability/server/utils/queries.ts @@ -83,6 +83,10 @@ export function rangeQuery( ]; } +export function existsQuery(field: string): QueryDslQueryContainer[] { + return [{ exists: { field } }]; +} + export function kqlQuery(kql?: string): estypes.QueryDslQueryContainer[] { if (!kql) { return []; From 23fa2e2e465706187031e91020369f51c877de8d Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Tue, 21 May 2024 17:10:28 +0200 Subject: [PATCH 02/11] Revert synthtrace changes --- .../src/scenarios/degraded_logs.ts | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts index d0afe09e3224f..cb5313ac70795 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts @@ -115,32 +115,6 @@ const scenario: Scenario = async (runOptions) => { .timestamp(timestamp); }; - const datasetSynth4Logs = (i: number, timestamp: number) => { - const index = Math.floor(Math.random() * 3); - const isMalformed = i % 10 === 0; - return log - .create() - .dataset('synth.3') - .message(MESSAGE_LOG_LEVELS[index].message as string) - .service(SERVICE_NAMES[index]) - .defaults({ - 'trace.id': generateShortId(), - 'agent.name': 'synth-agent', - 'orchestrator.cluster.name': CLUSTER[index].clusterName, - 'orchestrator.cluster.id': CLUSTER[index].clusterId, - 'orchestrator.resource.id': generateShortId(), - 'cloud.provider': CLOUD_PROVIDERS[Math.floor(Math.random() * 3)], - 'cloud.region': CLOUD_REGION[index], - 'cloud.availability_zone': isMalformed - ? MORE_THAN_1024_CHARS // "ignore_above": 1024 in mapping - : `${CLOUD_REGION[index]}a`, - 'cloud.project.id': generateShortId(), - 'cloud.instance.id': generateShortId(), - 'log.file.path': `/logs/${generateLongId()}/error.txt`, - }) - .timestamp(timestamp + 100); - }; - const logs = range .interval('1m') .rate(1) @@ -151,7 +125,6 @@ const scenario: Scenario = async (runOptions) => { datasetSynth1Logs(timestamp), datasetSynth2Logs(index, timestamp), datasetSynth3Logs(index, timestamp), - datasetSynth4Logs(index, timestamp), ]); }); From 07c0fc66e337cc3e102d0012220727dc734b3302 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Wed, 22 May 2024 18:06:12 +0200 Subject: [PATCH 03/11] Move pagination logic to state machine and added URL sync logic for shareable URLs --- .../dataset_quality/common/constants.ts | 5 ++ .../common/data_stream_details/errors.ts | 14 ++++ .../common/data_stream_details/index.ts | 8 ++ .../dataset_quality/common/types/index.ts | 2 + .../flyout/degraded_fields/table.tsx | 76 ++----------------- .../public/controller/public_state.ts | 4 +- .../public/controller/types.ts | 5 +- .../dataset_quality/public/hooks/index.ts | 1 + .../use_dataset_quality_degraded_field.ts | 72 ++++++++++++++++++ .../hooks/use_dataset_quality_flyout.tsx | 5 -- .../hooks/use_dataset_quality_table.tsx | 10 +-- .../data_stream_details_client.ts | 5 +- .../src/defaults.ts | 15 +++- .../src/notifications.ts | 9 --- .../src/state_machine.ts | 53 +++++++++++-- .../dataset_quality_controller/src/types.ts | 25 ++++-- .../data_streams/get_degraded_fields/index.ts | 3 +- .../dataset_quality/url_schema_v1.ts | 5 ++ 18 files changed, 204 insertions(+), 113 deletions(-) create mode 100644 x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/errors.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/index.ts create mode 100644 x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_degraded_field.ts diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts index d97def4d29625..ec2a9f6f641db 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/constants.ts @@ -17,6 +17,9 @@ export const DEGRADED_QUALITY_MINIMUM_PERCENTAGE = 0; export const DEFAULT_SORT_FIELD = 'title'; export const DEFAULT_SORT_DIRECTION = 'asc'; +export const DEFAULT_DEGRADED_FIELD_SORT_FIELD = 'count'; +export const DEFAULT_DEGRADED_FIELD_SORT_DIRECTION = 'desc'; + export const NONE = 'none'; export const DEFAULT_TIME_RANGE = { from: 'now-24h', to: 'now' }; @@ -33,3 +36,5 @@ export const NUMBER_FORMAT = '0,0.[000]'; export const BYTE_NUMBER_FORMAT = '0.0 b'; export const MAX_HOSTS_METRIC_VALUE = 50; + +export const MAX_DEGRADED_FIELDS = 1000; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/errors.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/errors.ts new file mode 100644 index 0000000000000..2ba8bac9f6b51 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/errors.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class GetDataStreamsDetailsError extends Error { + constructor(message: string) { + super(message); + Object.setPrototypeOf(this, new.target.prototype); + this.name = 'GetDataStreamsDetailsError'; + } +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/index.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/index.ts new file mode 100644 index 0000000000000..8761ef1b52a32 --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_stream_details/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './errors'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/types/index.ts b/x-pack/plugins/observability_solution/dataset_quality/common/types/index.ts index 143556e5a519f..bbdac062fde8d 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/types/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/types/index.ts @@ -7,3 +7,5 @@ export * from './dataset_types'; export * from './quality_types'; + +export type SortDirection = 'asc' | 'desc'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx index 5d242738f25ab..b559eb1acd147 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx @@ -6,84 +6,18 @@ */ import { EuiBasicTable, EuiEmptyPrompt } from '@elastic/eui'; -import React, { useMemo, useState } from 'react'; -import { orderBy } from 'lodash'; +import React from 'react'; +import { useDatasetQualityDegradedField } from '../../../hooks'; import { getDegradedFieldsColumns } from './columns'; -import { Direction, useDatasetQualityFlyout } from '../../../hooks'; import { flyoutDegradedFieldsTableLoadingText, flyoutDegradedFieldsTableNoData, } from '../../../../common/translations'; -import { DegradedField } from '../../../../common/api_types'; - -const DEFAULT_SORT_FIELD = 'count'; -const DEFAULT_SORT_DIRECTION = 'asc'; -const DEFAULT_ROWS_PER_PAGE = 10; - -interface TableOptions { - page: { - index: number; - size: number; - }; - sort: { - field: keyof DegradedField; - direction: 'asc' | 'desc'; - }; -} - -const DEFAULT_TABLE_OPTIONS: TableOptions = { - page: { - index: 0, - size: 0, - }, - sort: { - field: DEFAULT_SORT_FIELD, - direction: DEFAULT_SORT_DIRECTION, - }, -}; export const DegradedFieldTable = () => { - const { degradedFields, loadingState } = useDatasetQualityFlyout(); + const { loadingState, pagination, renderedItems, onTableChange, sort } = + useDatasetQualityDegradedField(); const columns = getDegradedFieldsColumns(); - const [tableOptions, setTableOptions] = useState(DEFAULT_TABLE_OPTIONS); - - const onTableChange = (options: { - page: { index: number; size: number }; - sort?: { field: keyof DegradedField; direction: Direction }; - }) => { - setTableOptions({ - page: { - index: options.page.index, - size: options.page.size, - }, - sort: { - field: options.sort?.field ?? DEFAULT_SORT_FIELD, - direction: options.sort?.direction ?? DEFAULT_SORT_DIRECTION, - }, - }); - }; - - const pagination = useMemo( - () => ({ - pageIndex: tableOptions.page.index, - pageSize: DEFAULT_ROWS_PER_PAGE, - totalItemCount: degradedFields?.length ?? 0, - hidePerPageOptions: true, - }), - [degradedFields, tableOptions] - ); - - const renderedItems = useMemo(() => { - const sortedItems = orderBy( - degradedFields, - tableOptions.sort.field, - tableOptions.sort.direction - ); - return sortedItems.slice( - tableOptions.page.index * DEFAULT_ROWS_PER_PAGE, - (tableOptions.page.index + 1) * DEFAULT_ROWS_PER_PAGE - ); - }, [degradedFields, tableOptions]); return ( { columns={columns} items={renderedItems ?? []} loading={loadingState.datasetDegradedFieldsLoading} - sorting={{ sort: tableOptions.sort }} + sorting={sort} onChange={onTableChange} pagination={pagination} noItemsMessage={ diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts b/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts index 138ea2cf2d4a6..22a37a60b6cc2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SortField } from '../hooks'; +import { DatasetTableSortField } from '../hooks'; import { DatasetQualityControllerContext, DEFAULT_CONTEXT, @@ -32,7 +32,7 @@ export const getContextFromPublicState = ( sort: publicState.table?.sort ? { ...publicState.table?.sort, - field: publicState.table?.sort.field as SortField, + field: publicState.table?.sort.field as DatasetTableSortField, } : DEFAULT_CONTEXT.table.sort, }, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts index a74b5ed498717..a3e211432b61e 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts @@ -24,7 +24,10 @@ export type DatasetQualityTableOptions = Partial< Omit & { sort: TableSortOptions } >; -export type DatasetQualityFlyoutOptions = Omit; +export type DatasetQualityFlyoutOptions = Omit< + WithFlyoutOptions['flyout'], + 'datasetDetails' | 'degradedFields' +> & { degradedFields: { table: DatasetQualityTableOptions } }; export type DatasetQualityFilterOptions = Partial; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts index 3d40c1674d40a..eafeddaeb1386 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/index.ts @@ -11,3 +11,4 @@ export * from './use_degraded_docs_chart'; export * from './use_link_to_logs_explorer'; export * from './use_summary_panel'; export * from './use_create_dataview'; +export * from './use_dataset_quality_degraded_field'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_degraded_field.ts b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_degraded_field.ts new file mode 100644 index 0000000000000..50b10562ce90a --- /dev/null +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_degraded_field.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { useSelector } from '@xstate/react'; +import { useCallback, useMemo } from 'react'; +import { orderBy } from 'lodash'; +import { useDatasetQualityContext } from '../components/dataset_quality/context'; +import { DegradedField } from '../../common/data_streams_stats'; +import { SortDirection } from '../../common/types'; +import { + DEFAULT_DEGRADED_FIELD_SORT_DIRECTION, + DEFAULT_DEGRADED_FIELD_SORT_FIELD, +} from '../../common/constants'; + +export type DegradedFieldSortField = keyof DegradedField; + +export function useDatasetQualityDegradedField() { + const { service } = useDatasetQualityContext(); + + const { degradedFields } = useSelector(service, (state) => state.context.flyout) ?? {}; + const { data, table } = degradedFields; + const { page, rowsPerPage, sort } = table; + + const pagination = { + pageIndex: page, + pageSize: rowsPerPage, + totalItemCount: data?.length ?? 0, + hidePerPageOptions: true, + }; + + const onTableChange = useCallback( + (options: { + page: { index: number; size: number }; + sort?: { field: DegradedFieldSortField; direction: SortDirection }; + }) => { + service.send({ + type: 'UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA', + degraded_field_criteria: { + page: options.page.index, + rowsPerPage: options.page.size, + sort: { + field: options.sort?.field || DEFAULT_DEGRADED_FIELD_SORT_FIELD, + direction: options.sort?.direction || DEFAULT_DEGRADED_FIELD_SORT_DIRECTION, + }, + }, + }); + }, + [service] + ); + + const renderedItems = useMemo(() => { + const sortedItems = orderBy(data, sort.field, sort.direction); + return sortedItems.slice(page * rowsPerPage, (page + 1) * rowsPerPage); + }, [data, sort.field, sort.direction, page, rowsPerPage]); + + const loadingState = useSelector(service, (state) => ({ + datasetDegradedFieldsLoading: state.matches( + 'flyout.initializing.dataStreamDegradedFields.fetching' + ), + })); + + return { + loadingState, + pagination, + onTableChange, + renderedItems, + sort: { sort }, + }; +} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx index 8e6ec7f2fd691..f653cb2631fc8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_flyout.tsx @@ -23,7 +23,6 @@ export const useDatasetQualityFlyout = () => { insightsTimeRange, breakdownField, isNonAggregatable, - degradedFields, } = useSelector(service, (state) => state.context.flyout) ?? {}; const { timeRange } = useSelector(service, (state) => state.context.filters); @@ -32,9 +31,6 @@ export const useDatasetQualityFlyout = () => { dataStreamDetailsLoading: state.matches('flyout.initializing.dataStreamDetails.fetching'), dataStreamSettingsLoading: state.matches('flyout.initializing.dataStreamSettings.fetching'), datasetIntegrationsLoading: state.matches('flyout.initializing.integrationDashboards.fetching'), - datasetDegradedFieldsLoading: state.matches( - 'flyout.initializing.dataStreamDegradedFields.fetching' - ), })); return { @@ -47,6 +43,5 @@ export const useDatasetQualityFlyout = () => { breakdownField, loadingState, flyoutLoading: !dataStreamStat, - degradedFields, }; }; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx index d9c936943cf0b..dd82b52a7e743 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx @@ -17,12 +17,12 @@ import { useDatasetQualityContext } from '../components/dataset_quality/context' import { FlyoutDataset } from '../state_machines/dataset_quality_controller'; import { useKibanaContextForPlugin } from '../utils'; import { filterInactiveDatasets, isActiveDataset } from '../utils/filter_inactive_datasets'; +import { SortDirection } from '../../common/types'; -export type Direction = 'asc' | 'desc'; -export type SortField = keyof DataStreamStat; +export type DatasetTableSortField = keyof DataStreamStat; const sortingOverrides: Partial<{ - [key in SortField]: SortField | ((item: DataStreamStat) => Primitive); + [key in DatasetTableSortField]: DatasetTableSortField | ((item: DataStreamStat) => Primitive); }> = { ['title']: 'name', ['size']: DataStreamStat.calculateFilteredSize, @@ -167,11 +167,11 @@ export const useDatasetQualityTable = () => { const onTableChange = useCallback( (options: { page: { index: number; size: number }; - sort?: { field: SortField; direction: Direction }; + sort?: { field: DatasetTableSortField; direction: SortDirection }; }) => { service.send({ type: 'UPDATE_TABLE_CRITERIA', - criteria: { + dataset_criteria: { page: options.page.index, rowsPerPage: options.page.size, sort: { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 33a05ef5fe57e..38d396e83386a 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -27,6 +27,7 @@ import { GetIntegrationDashboardsResponse, } from '../../../common/data_streams_stats'; import { IDataStreamDetailsClient } from './types'; +import { GetDataStreamsDetailsError } from '../../../common/data_stream_details'; export class DataStreamDetailsClient implements IDataStreamDetailsClient { constructor(private readonly http: HttpStart) {} @@ -83,7 +84,7 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { } ) .catch((error) => { - throw new GetDataStreamsStatsError( + throw new GetDataStreamsDetailsError( `Failed to fetch data stream degraded fields": ${error}` ); }); @@ -91,7 +92,7 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { return decodeOrThrow( getDataStreamDegradedFieldsResponseRt, (message: string) => - new GetDataStreamsStatsError( + new GetDataStreamsDetailsError( `Failed to decode data stream degraded fields response: ${message}"` ) )(response); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts index 72a7c2ddc24ee..a1b823d6f60fb 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts @@ -7,6 +7,8 @@ import { DEFAULT_DATASET_TYPE, + DEFAULT_DEGRADED_FIELD_SORT_DIRECTION, + DEFAULT_DEGRADED_FIELD_SORT_FIELD, DEFAULT_SORT_DIRECTION, DEFAULT_SORT_FIELD, } from '../../../../common/constants'; @@ -39,7 +41,18 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = { namespaces: [], qualities: [], }, - flyout: {}, + flyout: { + degradedFields: { + table: { + page: 0, + rowsPerPage: 10, + sort: { + field: DEFAULT_DEGRADED_FIELD_SORT_FIELD, + direction: DEFAULT_DEGRADED_FIELD_SORT_DIRECTION, + }, + }, + }, + }, datasets: [], isSizeStatsAvailable: true, nonAggregatableDatasets: [], diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts index bf77eaa722ad3..70da9d3d74e70 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/notifications.ts @@ -44,15 +44,6 @@ export const fetchDegradedStatsFailedNotifier = (toasts: IToasts, error: Error) }); }; -export const fetchDegradedFieldsFailedNotifier = (toasts: IToasts, error: Error) => { - toasts.addDanger({ - title: i18n.translate('xpack.datasetQuality.fetchDegradedFieldsFailed', { - defaultMessage: "We couldn't get your degraded fields information.", - }), - text: error.message, - }); -}; - export const fetchNonAggregatableDatasetsFailedNotifier = (toasts: IToasts, error: Error) => { toasts.addDanger({ title: i18n.translate('xpack.datasetQuality.fetchNonAggregatableDatasetsFailed', { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index 91fbf67374296..0c22fd947e196 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -35,7 +35,6 @@ import { fetchIntegrationsFailedNotifier, noDatasetSelected, fetchNonAggregatableDatasetsFailedNotifier, - fetchDegradedFieldsFailedNotifier, } from './notifications'; import { DatasetQualityControllerContext, @@ -44,6 +43,10 @@ import { DefaultDatasetQualityControllerState, FlyoutDataset, } from './types'; +import { + DEFAULT_DEGRADED_FIELD_SORT_DIRECTION, + DEFAULT_DEGRADED_FIELD_SORT_FIELD, +} from '../../../../common/constants'; export const createPureDatasetQualityControllerStateMachine = ( initialContext: DatasetQualityControllerContext @@ -324,7 +327,6 @@ export const createPureDatasetQualityControllerStateMachine = ( }, onError: { target: 'done', - actions: ['notifyFetchDegradedFieldsFailed'], }, }, }, @@ -332,6 +334,11 @@ export const createPureDatasetQualityControllerStateMachine = ( on: { UPDATE_INSIGHTS_TIME_RANGE: { target: 'fetching', + actions: ['resetDegradedFieldPage'], + }, + UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA: { + target: 'done', + actions: ['storeDegradedFieldTableOptions'], }, }, }, @@ -375,9 +382,22 @@ export const createPureDatasetQualityControllerStateMachine = ( { actions: { storeTableOptions: assign((_context, event) => { - return 'criteria' in event + return 'dataset_criteria' in event ? { - table: event.criteria, + table: event.dataset_criteria, + } + : {}; + }), + storeDegradedFieldTableOptions: assign((context, event) => { + return 'degraded_field_criteria' in event + ? { + flyout: { + ...context.flyout, + degradedFields: { + ...context.flyout.degradedFields, + table: event.degraded_field_criteria, + }, + }, } : {}; }), @@ -387,6 +407,22 @@ export const createPureDatasetQualityControllerStateMachine = ( page: 0, }, })), + resetDegradedFieldPage: assign((context, _event) => ({ + flyout: { + ...context.flyout, + degradedFields: { + ...context.flyout.degradedFields, + table: { + page: 0, + rowsPerPage: 10, + sort: { + field: DEFAULT_DEGRADED_FIELD_SORT_FIELD, + direction: DEFAULT_DEGRADED_FIELD_SORT_DIRECTION, + }, + }, + }, + }, + })), storeInactiveDatasetsVisibility: assign((context, _event) => { return { filters: { @@ -501,7 +537,10 @@ export const createPureDatasetQualityControllerStateMachine = ( ? { flyout: { ...context.flyout, - degradedFields: (event.data ?? {}) as DegradedField[], + degradedFields: { + ...context.flyout.degradedFields, + data: event.data as DegradedField[], + }, }, } : {}; @@ -615,8 +654,6 @@ export const createDatasetQualityControllerStateMachine = ({ fetchDatasetStatsFailedNotifier(toasts, event.data), notifyFetchDegradedStatsFailed: (_context, event: DoneInvokeEvent) => fetchDegradedStatsFailedNotifier(toasts, event.data), - notifyFetchDegradedFieldsFailed: (_context, event: DoneInvokeEvent) => - fetchDegradedFieldsFailedNotifier(toasts, event.data), notifyFetchNonAggregatableDatasetsFailed: (_context, event: DoneInvokeEvent) => fetchNonAggregatableDatasetsFailedNotifier(toasts, event.data), notifyFetchDatasetSettingsFailed: (_context, event: DoneInvokeEvent) => @@ -647,7 +684,7 @@ export const createDatasetQualityControllerStateMachine = ({ loadDegradedFieldsPerDataStream: (context) => { if (!context.flyout.dataset || !context.flyout.insightsTimeRange) { - fetchDatasetSettingsFailedNotifier(toasts, new Error(noDatasetSelected)); + fetchDatasetDetailsFailedNotifier(toasts, new Error(noDatasetSelected)); return Promise.resolve({}); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index 1096f38181262..2e995c6a50590 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -7,9 +7,9 @@ import { DoneInvokeEvent } from 'xstate'; import { RefreshInterval, TimeRange } from '@kbn/data-plugin/common'; -import { QualityIndicators } from '../../../../common/types'; +import { QualityIndicators, SortDirection } from '../../../../common/types'; import { Integration } from '../../../../common/data_streams_stats/integration'; -import { Direction, SortField } from '../../../hooks'; +import { DatasetTableSortField, DegradedFieldSortField } from '../../../hooks'; import { DegradedDocsStat } from '../../../../common/data_streams_stats/malformed_docs_stat'; import { DashboardType, @@ -29,15 +29,20 @@ export type FlyoutDataset = Omit< 'type' | 'size' | 'sizeBytes' | 'lastActivity' | 'degradedDocs' > & { type: string }; -interface TableCriteria { +interface TableCriteria { page: number; rowsPerPage: number; sort: { - field: SortField; - direction: Direction; + field: TSortField; + direction: SortDirection; }; } +interface DegradedFields { + table: TableCriteria; + data?: DegradedField[]; +} + export type TimeRangeConfig = Pick & { refresh: RefreshInterval; }; @@ -53,7 +58,7 @@ interface FiltersCriteria { } export interface WithTableOptions { - table: TableCriteria; + table: TableCriteria; } export interface WithFlyoutOptions { @@ -63,7 +68,7 @@ export interface WithFlyoutOptions { datasetDetails?: DataStreamDetails; insightsTimeRange?: TimeRangeConfig; breakdownField?: string; - degradedFields?: DegradedField[]; + degradedFields: DegradedFields; isNonAggregatable?: boolean; }; } @@ -156,7 +161,11 @@ export type DatasetQualityControllerContext = DatasetQualityControllerTypeState[ export type DatasetQualityControllerEvent = | { type: 'UPDATE_TABLE_CRITERIA'; - criteria: TableCriteria; + dataset_criteria: TableCriteria; + } + | { + type: 'UPDATE_DEGRADED_FIELDS_TABLE_CRITERIA'; + degraded_field_criteria: TableCriteria; } | { type: 'OPEN_FLYOUT'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts index 2d45769fc23b2..1262fa65c7229 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts @@ -9,7 +9,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { rangeQuery, termQuery, existsQuery } from '@kbn/observability-plugin/server'; import { DataStreamType } from '../../../../common/types'; import { DegradedField } from '../../../../common/api_types'; -import { DEFAULT_DATASET_TYPE } from '../../../../common/constants'; +import { DEFAULT_DATASET_TYPE, MAX_DEGRADED_FIELDS } from '../../../../common/constants'; import { createDatasetQualityESClient } from '../../../utils'; import { _IGNORED, @@ -48,6 +48,7 @@ export async function getDegradedFields({ const aggs = { degradedFields: { terms: { + size: MAX_DEGRADED_FIELDS, field: _IGNORED, }, aggs: { diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/common/url_schema/dataset_quality/url_schema_v1.ts b/x-pack/plugins/observability_solution/observability_logs_explorer/common/url_schema/dataset_quality/url_schema_v1.ts index 738b95d09ea6e..c97210b2a194b 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/common/url_schema/dataset_quality/url_schema_v1.ts +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/common/url_schema/dataset_quality/url_schema_v1.ts @@ -55,11 +55,16 @@ const timeRangeRT = rt.strict({ }), }); +const degradedFieldRT = rt.strict({ + table: tableRT, +}); + export const flyoutRT = rt.exact( rt.partial({ dataset: datasetRT, insightsTimeRange: timeRangeRT, breakdownField: rt.string, + degradedFields: degradedFieldRT, }) ); From f38aec507ba92501c314cf74fc4897552395693b Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Thu, 23 May 2024 12:03:20 +0200 Subject: [PATCH 04/11] Fix logic to only call specific datastream --- .../data_streams/get_degraded_fields/index.ts | 29 +++++-------------- .../server/routes/data_streams/routes.ts | 5 +--- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts index 1262fa65c7229..6bc0eb6a78a0a 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts @@ -6,42 +6,26 @@ */ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; -import { rangeQuery, termQuery, existsQuery } from '@kbn/observability-plugin/server'; -import { DataStreamType } from '../../../../common/types'; +import { rangeQuery, existsQuery } from '@kbn/observability-plugin/server'; import { DegradedField } from '../../../../common/api_types'; -import { DEFAULT_DATASET_TYPE, MAX_DEGRADED_FIELDS } from '../../../../common/constants'; +import { MAX_DEGRADED_FIELDS } from '../../../../common/constants'; import { createDatasetQualityESClient } from '../../../utils'; -import { - _IGNORED, - DATA_STREAM_DATASET, - DATA_STREAM_NAMESPACE, - DATA_STREAM_TYPE, - TIMESTAMP, -} from '../../../../common/es_fields'; +import { _IGNORED, TIMESTAMP } from '../../../../common/es_fields'; export async function getDegradedFields({ esClient, start, end, - type = DEFAULT_DATASET_TYPE, - namespace, - dataset, + dataStream, }: { esClient: ElasticsearchClient; start: number; end: number; - type?: DataStreamType; - namespace: string; - dataset: string; + dataStream: string; }): Promise { const datasetQualityESClient = createDatasetQualityESClient(esClient); - const filterQuery = [ - ...rangeQuery(start, end), - ...termQuery(DATA_STREAM_TYPE, type), - ...termQuery(DATA_STREAM_NAMESPACE, namespace), - ...termQuery(DATA_STREAM_DATASET, dataset), - ]; + const filterQuery = [...rangeQuery(start, end)]; const mustQuery = [...existsQuery(_IGNORED)]; @@ -62,6 +46,7 @@ export async function getDegradedFields({ }; const response = await datasetQualityESClient.search({ + index: dataStream, size: 0, query: { bool: { diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index c4731483d8cec..92e8b621f8aa0 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -142,13 +142,10 @@ const degradedFieldsRoute = createDatasetQualityServerRoute({ const coreContext = await context.core; const esClient = coreContext.elasticsearch.client.asCurrentUser; - const { type, dataset, namespace } = indexNameToDataStreamParts(dataStream); return await getDegradedFields({ esClient, - type, - dataset, - namespace, + dataStream, ...params.query, }); }, From 592c396e446168df6b4042eddde1f490ea90e1b8 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Thu, 23 May 2024 15:37:13 +0200 Subject: [PATCH 05/11] Add API tests --- .../data_streams/degraded_fields.spec.ts | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts new file mode 100644 index 0000000000000..55233db91565b --- /dev/null +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { log, timerange } from '@kbn/apm-synthtrace-client'; +import expect from '@kbn/expect'; +import { DegradedField } from '@kbn/dataset-quality-plugin/common/api_types'; +import { DatasetQualityApiClientKey } from '../../common/config'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +const MORE_THAN_1024_CHARS = + 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur?'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const registry = getService('registry'); + const synthtrace = getService('logSynthtraceEsClient'); + const datasetQualityApiClient = getService('datasetQualityApiClient'); + const start = '2024-05-22T08:00:00.000Z'; + const end = '2024-05-23T08:02:00.000Z'; + const type = 'logs'; + const dataset = 'nginx.access'; + const degradedFieldDataset = 'nginx.error'; + const namespace = 'default'; + const serviceName = 'my-service'; + const hostName = 'synth-host'; + + async function callApiAs(user: DatasetQualityApiClientKey, dataStream: string) { + return await datasetQualityApiClient[user]({ + endpoint: 'GET /internal/dataset_quality/data_streams/{dataStream}/degraded_fields', + params: { + path: { + dataStream, + }, + query: { + start, + end, + }, + }, + }); + } + + registry.when('Degraded Fields per DataStream', { config: 'basic' }, () => { + describe('gets the degraded fields per data stream', () => { + before(async () => { + await synthtrace.index([ + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a log message') + .timestamp(timestamp) + .dataset(dataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/my-service.log', + 'service.name': serviceName + 0, + 'host.name': hostName, + }) + ), + timerange(start, end) + .interval('1m') + .rate(1) + .generator((timestamp) => + log + .create() + .message('This is a error message') + .logLevel(MORE_THAN_1024_CHARS) + .timestamp(timestamp) + .dataset(degradedFieldDataset) + .namespace(namespace) + .defaults({ + 'log.file.path': '/error.log', + 'service.name': serviceName + 1, + 'trace.id': MORE_THAN_1024_CHARS, + }) + ), + ]); + }); + + after(async () => { + await synthtrace.clean(); + }); + + it('returns no results when dataStream does not have any degraded fields', async () => { + const resp = await callApiAs('datasetQualityLogsUser', `${type}-${dataset}-${namespace}`); + expect(resp.body.length).to.be(0); + }); + + it('returns results when dataStream do have degraded fields', async () => { + const expectedDegradedFields = ['log.level', 'trace.id']; + const resp = await callApiAs( + 'datasetQualityLogsUser', + `${type}-${degradedFieldDataset}-${namespace}` + ); + const degradedFields = resp.body.map((field: DegradedField) => field.fieldName); + + expect(resp.body.length).to.be(2); + expect(degradedFields).to.eql(expectedDegradedFields); + }); + }); + }); +} From 0402cb903240bf6449d2aac5a3ee521c6b9d3c7c Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Fri, 24 May 2024 15:07:33 +0200 Subject: [PATCH 06/11] Add FTR tests --- .../dataset_quality/common/translations.ts | 2 +- .../flyout/degraded_fields/table.tsx | 6 +- .../src/state_machine.ts | 2 +- .../apps/dataset_quality/data/logs_data.ts | 50 ++ .../dataset_quality/dataset_quality_flyout.ts | 729 ++++++++++------- .../page_objects/dataset_quality.ts | 41 +- .../dataset_quality/data/logs_data.ts | 50 ++ .../dataset_quality/dataset_quality_flyout.ts | 757 ++++++++++-------- 8 files changed, 1002 insertions(+), 635 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index 4944e8a054a53..47d1aff3183d7 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -147,7 +147,7 @@ export const flyoutImprovementText = i18n.translate( export const flyoutImprovementTooltip = i18n.translate( 'xpack.datasetQuality.flyoutDegradedFieldsSectionTooltip', { - defaultMessage: 'Some logical tooltip for Improvements', + defaultMessage: 'Set of degraded fields in the dataset.', } ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx index b559eb1acd147..419095865e5d1 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx @@ -28,12 +28,16 @@ export const DegradedFieldTable = () => { sorting={sort} onChange={onTableChange} pagination={pagination} + data-test-subj="datasetQualityFlyoutDegradedFieldTable" + rowProps={{ + 'data-test-subj': 'datasetQualityFlyoutDegradedTableRow', + }} noItemsMessage={ loadingState.datasetDegradedFieldsLoading ? ( flyoutDegradedFieldsTableLoadingText ) : ( {flyoutDegradedFieldsTableNoData}} hasBorder={false} diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index 0c22fd947e196..802a5b8e7d66a 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -510,7 +510,7 @@ export const createPureDatasetQualityControllerStateMachine = ( }, }; }), - resetFlyoutOptions: assign((_context, _event) => ({ flyout: undefined })), + resetFlyoutOptions: assign((_context, _event) => ({ flyout: DEFAULT_CONTEXT.flyout })), storeDataStreamStats: assign((_context, event) => { if ('data' in event) { const dataStreamStats = event.data as DataStreamStat[]; diff --git a/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts b/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts index 68d96070990f0..399030d1dd377 100644 --- a/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts +++ b/x-pack/test/functional/apps/dataset_quality/data/logs_data.ts @@ -139,6 +139,56 @@ export function createLogRecord( .timestamp(timestamp); } +/* +The helped function generates 2 sets of Malformed Docs for the given dataset. +1 set has more Malformed fields than the second one. This help in having different counts and hence sorting + */ +export function createDegradedFieldsRecord({ + to, + count = 1, + dataset, +}: { + to: string; + count?: number; + dataset: string; +}) { + return timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(count) + .fill(0) + .flatMap((_, index) => [ + log + .create() + .dataset(dataset) + .message(MESSAGE_LOG_LEVELS[0].message) + .logLevel(MORE_THAN_1024_CHARS) + .service(SERVICE_NAMES[0]) + .namespace(defaultNamespace) + .defaults({ + 'trace.id': generateShortId(), + 'agent.name': 'synth-agent', + 'cloud.availability_zone': MORE_THAN_1024_CHARS, + }) + .timestamp(timestamp), + log + .create() + .dataset(dataset) + .message(MESSAGE_LOG_LEVELS[1].message) + .logLevel(MESSAGE_LOG_LEVELS[1].level) + .service(SERVICE_NAMES[0]) + .namespace(defaultNamespace) + .defaults({ + 'trace.id': generateShortId(), + 'agent.name': 'synth-agent', + 'cloud.availability_zone': MORE_THAN_1024_CHARS, + }) + .timestamp(timestamp), + ]); + }); +} + export const datasetNames = ['synth.1', 'synth.2', 'synth.3']; export const defaultNamespace = 'default'; diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts index 8e1abbbefa9dc..5a3902aafc7aa 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts @@ -7,7 +7,12 @@ import expect from '@kbn/expect'; import { DatasetQualityFtrProviderContext } from './config'; -import { datasetNames, getInitialTestLogs, getLogsForDataset } from './data'; +import { + createDegradedFieldsRecord, + datasetNames, + getInitialTestLogs, + getLogsForDataset, +} from './data'; const integrationActions = { overview: 'Overview', @@ -28,414 +33,520 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid const browser = getService('browser'); const to = '2024-01-01T12:00:00.000Z'; - // FLAKY: https://github.com/elastic/kibana/issues/182154 - describe.skip('Dataset quality flyout', () => { - before(async () => { - await synthtrace.index(getInitialTestLogs({ to, count: 4 })); - await PageObjects.datasetQuality.navigateTo(); - }); + describe('Dataset quality flyout', () => { + // FLAKY: https://github.com/elastic/kibana/issues/182154 + // Added this sub describe block so that the existing flaky tests can be skipped and new ones can be added in the other describe block + describe.skip('Other dataset quality flyout tests', () => { + before(async () => { + await synthtrace.index(getInitialTestLogs({ to, count: 4 })); + await PageObjects.datasetQuality.navigateTo(); + }); - after(async () => { - await synthtrace.clean(); - await PageObjects.observabilityLogsExplorer.removeInstalledPackages(); - }); + after(async () => { + await synthtrace.clean(); + await PageObjects.observabilityLogsExplorer.removeInstalledPackages(); + }); - it('opens the flyout for the right dataset', async () => { - const testDatasetName = datasetNames[1]; + it('opens the flyout for the right dataset', async () => { + const testDatasetName = datasetNames[1]; - await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); + await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutTitle - ); - }); + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutTitle + ); + }); - it('shows the correct last activity', async () => { - const testDatasetName = datasetNames[0]; + it('shows the correct last activity', async () => { + const testDatasetName = datasetNames[0]; - // Update last activity for the dataset - await PageObjects.datasetQuality.closeFlyout(); - await synthtrace.index( - getLogsForDataset({ to: new Date().toISOString(), count: 1, dataset: testDatasetName }) - ); - await PageObjects.datasetQuality.refreshTable(); + // Update last activity for the dataset + await PageObjects.datasetQuality.closeFlyout(); + await synthtrace.index( + getLogsForDataset({ to: new Date().toISOString(), count: 1, dataset: testDatasetName }) + ); + await PageObjects.datasetQuality.refreshTable(); - const cols = await PageObjects.datasetQuality.parseDatasetTable(); + const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const datasetNameCol = cols['Dataset Name']; - const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); + const datasetNameCol = cols['Dataset Name']; + const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); - const testDatasetRowIndex = datasetNameColCellTexts.findIndex( - (dName: string) => dName === testDatasetName - ); + const testDatasetRowIndex = datasetNameColCellTexts.findIndex( + (dName: string) => dName === testDatasetName + ); - const lastActivityText = (await cols['Last Activity'].getCellTexts())[testDatasetRowIndex]; + const lastActivityText = (await cols['Last Activity'].getCellTexts())[testDatasetRowIndex]; - await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); + await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); - const lastActivityTextExists = await PageObjects.datasetQuality.doestTextExistInFlyout( - lastActivityText, - `[data-test-subj=${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutFieldValue}]` - ); + const lastActivityTextExists = await PageObjects.datasetQuality.doestTextExistInFlyout( + lastActivityText, + `[data-test-subj=${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutFieldValue}]` + ); - expect(lastActivityTextExists).to.eql(true); - }); + expect(lastActivityTextExists).to.eql(true); + }); - it('reflects the breakdown field state in url', async () => { - const testDatasetName = datasetNames[0]; - await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); + it('reflects the breakdown field state in url', async () => { + const testDatasetName = datasetNames[0]; + await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); - const breakdownField = 'service.name'; - await PageObjects.datasetQuality.selectBreakdownField(breakdownField); + const breakdownField = 'service.name'; + await PageObjects.datasetQuality.selectBreakdownField(breakdownField); - // Wait for URL to contain "breakdownField:service.name" - await retry.tryForTime(5000, async () => { - const currentUrl = await browser.getCurrentUrl(); - expect(decodeURIComponent(currentUrl)).to.contain(`breakdownField:${breakdownField}`); - }); + // Wait for URL to contain "breakdownField:service.name" + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + expect(decodeURIComponent(currentUrl)).to.contain(`breakdownField:${breakdownField}`); + }); - // Clear breakdown field - await PageObjects.datasetQuality.selectBreakdownField('No breakdown'); + // Clear breakdown field + await PageObjects.datasetQuality.selectBreakdownField('No breakdown'); - // Wait for URL to not contain "breakdownField" - await retry.tryForTime(5000, async () => { - const currentUrl = await browser.getCurrentUrl(); - expect(currentUrl).to.not.contain('breakdownField'); + // Wait for URL to not contain "breakdownField" + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + expect(currentUrl).to.not.contain('breakdownField'); + }); }); - }); - it('shows the integration details', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; - const apacheIntegrationId = 'apache'; + it('shows the integration details', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + const apacheIntegrationId = 'apache'; - await PageObjects.observabilityLogsExplorer.navigateTo(); + await PageObjects.observabilityLogsExplorer.navigateTo(); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + // Add initial integrations + await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); - // Index 10 logs for `logs-apache.access` dataset - await synthtrace.index( - getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) - ); + // Index 10 logs for `logs-apache.access` dataset + await synthtrace.index( + getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) + ); - await PageObjects.datasetQuality.navigateTo(); + await PageObjects.datasetQuality.navigateTo(); - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - const integrationNameElements = await PageObjects.datasetQuality.getFlyoutElementsByText( - '[data-test-subj=datasetQualityFlyoutFieldValue]', - apacheIntegrationId - ); + const integrationNameElements = await PageObjects.datasetQuality.getFlyoutElementsByText( + '[data-test-subj=datasetQualityFlyoutFieldValue]', + apacheIntegrationId + ); - await PageObjects.datasetQuality.closeFlyout(); + await PageObjects.datasetQuality.closeFlyout(); - expect(integrationNameElements.length).to.eql(1); - }); + expect(integrationNameElements.length).to.eql(1); + }); - it('goes to log explorer page when open button is clicked', async () => { - const testDatasetName = datasetNames[2]; - await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); + it('goes to log explorer page when open button is clicked', async () => { + const testDatasetName = datasetNames[2]; + await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); - await (await PageObjects.datasetQuality.getFlyoutLogsExplorerButton()).click(); + await (await PageObjects.datasetQuality.getFlyoutLogsExplorerButton()).click(); - // Confirm dataset selector text in observability logs explorer - const datasetSelectorText = - await PageObjects.observabilityLogsExplorer.getDataSourceSelectorButtonText(); - expect(datasetSelectorText).to.eql(testDatasetName); - }); + // Confirm dataset selector text in observability logs explorer + const datasetSelectorText = + await PageObjects.observabilityLogsExplorer.getDataSourceSelectorButtonText(); + expect(datasetSelectorText).to.eql(testDatasetName); + }); - it('shows summary KPIs', async () => { - await PageObjects.datasetQuality.navigateTo(); + it('shows summary KPIs', async () => { + await PageObjects.datasetQuality.navigateTo(); - const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - const summary = await PageObjects.datasetQuality.parseFlyoutKpis(); - expect(summary).to.eql({ - docsCountTotal: '0', - size: '0.0 B', - services: '0', - hosts: '0', - degradedDocs: '0', + const summary = await PageObjects.datasetQuality.parseFlyoutKpis(); + expect(summary).to.eql({ + docsCountTotal: '0', + size: '0.0 B', + services: '0', + hosts: '0', + degradedDocs: '0', + }); }); - }); - it('shows the updated KPIs', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - - const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis(); - - // Set time range to 3 days ago - const flyoutBodyContainer = await testSubjects.find( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutBody - ); - await PageObjects.datasetQuality.setDatePickerLastXUnits(flyoutBodyContainer, 3, 'd'); - - // Index 2 doc 2 days ago - const time2DaysAgo = Date.now() - 2 * 24 * 60 * 60 * 1000; - await synthtrace.index( - getLogsForDataset({ - to: time2DaysAgo, - count: 2, - dataset: apacheAccessDatasetName, - isMalformed: false, - }) - ); - - // Index 5 degraded docs 2 days ago - await synthtrace.index( - getLogsForDataset({ - to: time2DaysAgo, - count: 5, - dataset: apacheAccessDatasetName, - isMalformed: true, - }) - ); - - await PageObjects.datasetQuality.refreshFlyout(); - const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis(); - - expect(parseInt(summaryAfter.docsCountTotal, 10)).to.be.greaterThan( - parseInt(summaryBefore.docsCountTotal, 10) - ); - - expect(parseInt(summaryAfter.degradedDocs, 10)).to.be.greaterThan( - parseInt(summaryBefore.degradedDocs, 10) - ); - - expect(parseInt(summaryAfter.size, 10)).to.be.greaterThan(parseInt(summaryBefore.size, 10)); - expect(parseInt(summaryAfter.services, 10)).to.be.greaterThan( - parseInt(summaryBefore.services, 10) - ); - expect(parseInt(summaryAfter.hosts, 10)).to.be.greaterThan(parseInt(summaryBefore.hosts, 10)); - }); + it('shows the updated KPIs', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - it('shows the right number of services', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - - const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis(); - const testServices = ['test-srv-1', 'test-srv-2']; - - // Index 2 docs with different services - const timeNow = Date.now(); - await synthtrace.index( - getLogsForDataset({ - to: timeNow, - count: 2, - dataset: apacheAccessDatasetName, - isMalformed: false, - services: testServices, - }) - ); - - await PageObjects.datasetQuality.refreshFlyout(); - const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis(); - - expect(parseInt(summaryAfter.services, 10)).to.eql( - parseInt(summaryBefore.services, 10) + testServices.length - ); - }); + const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis(); - it('goes to log explorer for degraded docs when show all is clicked', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + // Set time range to 3 days ago + const flyoutBodyContainer = await testSubjects.find( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutBody + ); + await PageObjects.datasetQuality.setDatePickerLastXUnits(flyoutBodyContainer, 3, 'd'); + + // Index 2 doc 2 days ago + const time2DaysAgo = Date.now() - 2 * 24 * 60 * 60 * 1000; + await synthtrace.index( + getLogsForDataset({ + to: time2DaysAgo, + count: 2, + dataset: apacheAccessDatasetName, + isMalformed: false, + }) + ); - const degradedDocsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.degradedDocs}`; - await testSubjects.click(degradedDocsShowAllSelector); - await browser.switchTab(1); + // Index 5 degraded docs 2 days ago + await synthtrace.index( + getLogsForDataset({ + to: time2DaysAgo, + count: 5, + dataset: apacheAccessDatasetName, + isMalformed: true, + }) + ); - // Confirm dataset selector text in observability logs explorer - const datasetSelectorText = - await PageObjects.observabilityLogsExplorer.getDataSourceSelectorButtonText(); - expect(datasetSelectorText).to.contain(apacheAccessDatasetName); + await PageObjects.datasetQuality.refreshFlyout(); + const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis(); - await browser.closeCurrentWindow(); - await browser.switchTab(0); - }); + expect(parseInt(summaryAfter.docsCountTotal, 10)).to.be.greaterThan( + parseInt(summaryBefore.docsCountTotal, 10) + ); + + expect(parseInt(summaryAfter.degradedDocs, 10)).to.be.greaterThan( + parseInt(summaryBefore.degradedDocs, 10) + ); - // Blocked by https://github.com/elastic/kibana/issues/181705 - it.skip('goes to infra hosts for hosts when show all is clicked', async () => { - const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + expect(parseInt(summaryAfter.size, 10)).to.be.greaterThan(parseInt(summaryBefore.size, 10)); + expect(parseInt(summaryAfter.services, 10)).to.be.greaterThan( + parseInt(summaryBefore.services, 10) + ); + expect(parseInt(summaryAfter.hosts, 10)).to.be.greaterThan( + parseInt(summaryBefore.hosts, 10) + ); + }); - const hostsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.hosts}`; - await testSubjects.click(hostsShowAllSelector); - await browser.switchTab(1); + it('shows the right number of services', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis(); + const testServices = ['test-srv-1', 'test-srv-2']; + + // Index 2 docs with different services + const timeNow = Date.now(); + await synthtrace.index( + getLogsForDataset({ + to: timeNow, + count: 2, + dataset: apacheAccessDatasetName, + isMalformed: false, + services: testServices, + }) + ); + + await PageObjects.datasetQuality.refreshFlyout(); + const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis(); - // Confirm url contains metrics/hosts - await retry.tryForTime(5000, async () => { - const currentUrl = await browser.getCurrentUrl(); - const parsedUrl = new URL(currentUrl); - expect(parsedUrl.pathname).to.contain('/app/metrics/hosts'); + expect(parseInt(summaryAfter.services, 10)).to.eql( + parseInt(summaryBefore.services, 10) + testServices.length + ); }); - await browser.closeCurrentWindow(); - await browser.switchTab(0); - }); + it('goes to log explorer for degraded docs when show all is clicked', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + + const degradedDocsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.degradedDocs}`; + await testSubjects.click(degradedDocsShowAllSelector); + await browser.switchTab(1); - it('Integration actions menu is present with correct actions', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; + // Confirm dataset selector text in observability logs explorer + const datasetSelectorText = + await PageObjects.observabilityLogsExplorer.getDataSourceSelectorButtonText(); + expect(datasetSelectorText).to.contain(apacheAccessDatasetName); - await PageObjects.observabilityLogsExplorer.navigateTo(); + await browser.closeCurrentWindow(); + await browser.switchTab(0); + }); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + // Blocked by https://github.com/elastic/kibana/issues/181705 + it.skip('goes to infra hosts for hosts when show all is clicked', async () => { + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - // Index 10 logs for `logs-apache.access` dataset - await synthtrace.index( - getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) - ); + const hostsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.hosts}`; + await testSubjects.click(hostsShowAllSelector); + await browser.switchTab(1); - await PageObjects.datasetQuality.navigateTo(); + // Confirm url contains metrics/hosts + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + const parsedUrl = new URL(currentUrl); + expect(parsedUrl.pathname).to.contain('/app/metrics/hosts'); + }); - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - await PageObjects.datasetQuality.openIntegrationActionsMenu(); + await browser.closeCurrentWindow(); + await browser.switchTab(0); + }); - const actions = await Promise.all( - Object.values(integrationActions).map((action) => - PageObjects.datasetQuality.getIntegrationActionButtonByAction(action) - ) - ); + it('Integration actions menu is present with correct actions', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; - expect(actions.length).to.eql(3); - }); + await PageObjects.observabilityLogsExplorer.navigateTo(); - it('Integration dashboard action hidden for integrations without dashboards', async () => { - const bitbucketDatasetName = 'atlassian_bitbucket.audit'; - const bitbucketDatasetHumanName = 'Bitbucket Audit Logs'; + // Add initial integrations + await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); - await PageObjects.observabilityLogsExplorer.navigateTo(); + // Index 10 logs for `logs-apache.access` dataset + await synthtrace.index( + getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) + ); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.installPackage({ - name: 'atlassian_bitbucket', - version: '1.14.0', + await PageObjects.datasetQuality.navigateTo(); + + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + await PageObjects.datasetQuality.openIntegrationActionsMenu(); + + const actions = await Promise.all( + Object.values(integrationActions).map((action) => + PageObjects.datasetQuality.getIntegrationActionButtonByAction(action) + ) + ); + + expect(actions.length).to.eql(3); }); - // Index 10 logs for `atlassian_bitbucket.audit` dataset - await synthtrace.index(getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName })); + it('Integration dashboard action hidden for integrations without dashboards', async () => { + const bitbucketDatasetName = 'atlassian_bitbucket.audit'; + const bitbucketDatasetHumanName = 'Bitbucket Audit Logs'; - await PageObjects.datasetQuality.navigateTo(); + await PageObjects.observabilityLogsExplorer.navigateTo(); - await PageObjects.datasetQuality.openDatasetFlyout(bitbucketDatasetHumanName); - await PageObjects.datasetQuality.openIntegrationActionsMenu(); + // Add initial integrations + await PageObjects.observabilityLogsExplorer.installPackage({ + name: 'atlassian_bitbucket', + version: '1.14.0', + }); - await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutIntegrationAction( - integrationActions.viewDashboards - ) - ); - }); + // Index 10 logs for `atlassian_bitbucket.audit` dataset + await synthtrace.index(getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName })); - it('Integration overview action should navigate to the integration overview page', async () => { - const bitbucketDatasetName = 'atlassian_bitbucket.audit'; - const bitbucketDatasetHumanName = 'Bitbucket Audit Logs'; + await PageObjects.datasetQuality.navigateTo(); - await PageObjects.observabilityLogsExplorer.navigateTo(); + await PageObjects.datasetQuality.openDatasetFlyout(bitbucketDatasetHumanName); + await PageObjects.datasetQuality.openIntegrationActionsMenu(); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.installPackage({ - name: 'atlassian_bitbucket', - version: '1.14.0', + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutIntegrationAction( + integrationActions.viewDashboards + ) + ); }); - // Index 10 logs for `atlassian_bitbucket.audit` dataset - await synthtrace.index(getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName })); + it('Integration overview action should navigate to the integration overview page', async () => { + const bitbucketDatasetName = 'atlassian_bitbucket.audit'; + const bitbucketDatasetHumanName = 'Bitbucket Audit Logs'; + + await PageObjects.observabilityLogsExplorer.navigateTo(); - await PageObjects.datasetQuality.navigateTo(); + // Add initial integrations + await PageObjects.observabilityLogsExplorer.installPackage({ + name: 'atlassian_bitbucket', + version: '1.14.0', + }); - await PageObjects.datasetQuality.openDatasetFlyout(bitbucketDatasetHumanName); - await PageObjects.datasetQuality.openIntegrationActionsMenu(); + // Index 10 logs for `atlassian_bitbucket.audit` dataset + await synthtrace.index(getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName })); - const action = await PageObjects.datasetQuality.getIntegrationActionButtonByAction( - integrationActions.overview - ); + await PageObjects.datasetQuality.navigateTo(); + + await PageObjects.datasetQuality.openDatasetFlyout(bitbucketDatasetHumanName); + await PageObjects.datasetQuality.openIntegrationActionsMenu(); + + const action = await PageObjects.datasetQuality.getIntegrationActionButtonByAction( + integrationActions.overview + ); - await action.click(); + await action.click(); - await retry.tryForTime(5000, async () => { - const currentUrl = await browser.getCurrentUrl(); - const parsedUrl = new URL(currentUrl); + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + const parsedUrl = new URL(currentUrl); - expect(parsedUrl.pathname).to.contain('/app/integrations/detail/atlassian_bitbucket'); + expect(parsedUrl.pathname).to.contain('/app/integrations/detail/atlassian_bitbucket'); + }); }); - }); - it('Integration template action should navigate to the index template page', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; + it('Integration template action should navigate to the index template page', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + + await PageObjects.observabilityLogsExplorer.navigateTo(); + + // Add initial integrations + await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + + // Index 10 logs for `logs-apache.access` dataset + await synthtrace.index( + getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) + ); + + await PageObjects.datasetQuality.navigateTo(); + + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + await PageObjects.datasetQuality.openIntegrationActionsMenu(); + + await retry.tryForTime(5000, async () => { + const action = await PageObjects.datasetQuality.getIntegrationActionButtonByAction( + integrationActions.template + ); + + await action.click(); + + const currentUrl = await browser.getCurrentUrl(); + const parsedUrl = new URL(currentUrl); + expect(parsedUrl.pathname).to.contain( + `/app/management/data/index_management/templates/logs-${apacheAccessDatasetName}` + ); + }); + }); + + it('Integration dashboard action should navigate to the selected dashboard', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.observabilityLogsExplorer.navigateTo(); + await PageObjects.observabilityLogsExplorer.navigateTo(); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + // Add initial integrations + await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); - // Index 10 logs for `logs-apache.access` dataset - await synthtrace.index( - getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) - ); + // Index 10 logs for `logs-apache.access` dataset + await synthtrace.index( + getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) + ); - await PageObjects.datasetQuality.navigateTo(); + await PageObjects.datasetQuality.navigateTo(); - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - await PageObjects.datasetQuality.openIntegrationActionsMenu(); + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + await PageObjects.datasetQuality.openIntegrationActionsMenu(); - await retry.tryForTime(5000, async () => { const action = await PageObjects.datasetQuality.getIntegrationActionButtonByAction( - integrationActions.template + integrationActions.viewDashboards ); await action.click(); - const currentUrl = await browser.getCurrentUrl(); - const parsedUrl = new URL(currentUrl); - expect(parsedUrl.pathname).to.contain( - `/app/management/data/index_management/templates/logs-${apacheAccessDatasetName}` - ); + const dashboardButtons = await PageObjects.datasetQuality.getIntegrationDashboardButtons(); + const firstDashboardButton = await dashboardButtons[0]; + const dashboardText = await firstDashboardButton.getVisibleText(); + + await firstDashboardButton.click(); + + const breadcrumbText = await testSubjects.getVisibleText('breadcrumb last'); + + expect(breadcrumbText).to.eql(dashboardText); }); }); - it('Integration dashboard action should navigate to the selected dashboard', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; + // The above describe block has some failing/flaky tests which will + // be fixed as part of the tech debt mentioned here + // https://github.com/elastic/kibana/issues/184145 + // Until then, the below describe block is added to cover the tests for the + // newly added degraded Fields Table. This must be merged under the above + // describe block once the tech debt is fixed. + describe('Dataset quality flyout with degraded fields', () => { + const goodDatasetName = 'good'; + const degradedDatasetName = 'degraded'; + const today = Date.now(); + before(async () => { + await synthtrace.index([ + getLogsForDataset({ + to: today, + count: 2, + dataset: goodDatasetName, + isMalformed: false, + }), + createDegradedFieldsRecord({ + to: today, + count: 2, + dataset: degradedDatasetName, + }), + ]); + await PageObjects.datasetQuality.navigateTo(); + }); + + after(async () => { + await synthtrace.clean(); + }); + + it('shows the degraded fields table with no data when no degraded fields are present', async () => { + await PageObjects.datasetQuality.openDatasetFlyout(goodDatasetName); - await PageObjects.observabilityLogsExplorer.navigateTo(); + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedTableNoData + ); + + await PageObjects.datasetQuality.closeFlyout(); + }); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + it('should load the degraded fields table with data', async () => { + await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName); - // Index 10 logs for `logs-apache.access` dataset - await synthtrace.index( - getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) - ); + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedFieldTable + ); + + const rows = + await PageObjects.datasetQuality.getDatasetQualityFlyoutDegradedFieldTableRows(); - await PageObjects.datasetQuality.navigateTo(); + expect(rows.length).to.eql(2); - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - await PageObjects.datasetQuality.openIntegrationActionsMenu(); + await PageObjects.datasetQuality.closeFlyout(); + }); - const action = await PageObjects.datasetQuality.getIntegrationActionButtonByAction( - integrationActions.viewDashboards - ); + it('should sort the table when the count table header is clicked', async () => { + await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName); - await action.click(); + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); - const dashboardButtons = await PageObjects.datasetQuality.getIntegrationDashboardButtons(); - const firstDashboardButton = await dashboardButtons[0]; - const dashboardText = await firstDashboardButton.getVisibleText(); + const countColumn = table.Count; + const cellTexts = await countColumn.getCellTexts(); - await firstDashboardButton.click(); + await countColumn.sort('ascending'); + const sortedCellTexts = await countColumn.getCellTexts(); - const breadcrumbText = await testSubjects.getVisibleText('breadcrumb last'); + expect(cellTexts.reverse()).to.eql(sortedCellTexts); - expect(breadcrumbText).to.eql(dashboardText); + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should update the table when new data is ingested and the flyout is refreshed using the time selector', async () => { + await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName); + + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); + + const countColumn = table.Count; + const cellTexts = await countColumn.getCellTexts(); + + await synthtrace.index([ + createDegradedFieldsRecord({ + to: today, + count: 2, + dataset: degradedDatasetName, + }), + ]); + + await PageObjects.datasetQuality.refreshFlyout(); + + const updatedCellTexts = await countColumn.getCellTexts(); + + const singleValuePreviously = parseInt(cellTexts[0], 10); + const singleValueNow = parseInt(updatedCellTexts[0], 10); + + expect(singleValueNow).to.be(singleValuePreviously * 2); + + await PageObjects.datasetQuality.closeFlyout(); + }); }); }); } diff --git a/x-pack/test/functional/page_objects/dataset_quality.ts b/x-pack/test/functional/page_objects/dataset_quality.ts index f81cffea160f8..ba7a4241bdf8a 100644 --- a/x-pack/test/functional/page_objects/dataset_quality.ts +++ b/x-pack/test/functional/page_objects/dataset_quality.ts @@ -14,6 +14,10 @@ import { OBSERVABILITY_DATASET_QUALITY_URL_STATE_KEY, datasetQualityUrlSchemaV1, } from '@kbn/observability-logs-explorer-plugin/common'; +import { + DEFAULT_DEGRADED_FIELD_SORT_DIRECTION, + DEFAULT_DEGRADED_FIELD_SORT_FIELD, +} from '@kbn/dataset-quality-plugin/common/constants'; import { FtrProviderContext } from '../ftr_provider_context'; const defaultPageState: datasetQualityUrlSchemaV1.UrlSchema = { @@ -22,7 +26,18 @@ const defaultPageState: datasetQualityUrlSchemaV1.UrlSchema = { page: 0, }, filters: {}, - flyout: {}, + flyout: { + degradedFields: { + table: { + page: 0, + rowsPerPage: 10, + sort: { + field: DEFAULT_DEGRADED_FIELD_SORT_FIELD, + direction: DEFAULT_DEGRADED_FIELD_SORT_DIRECTION, + }, + }, + }, + }, }; type SummaryPanelKpi = Record< @@ -60,6 +75,8 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv datasetQualityFlyout: 'datasetQualityFlyout', datasetQualityFlyoutBody: 'datasetQualityFlyoutBody', datasetQualityFlyoutTitle: 'datasetQualityFlyoutTitle', + datasetQualityFlyoutDegradedFieldTable: 'datasetQualityFlyoutDegradedFieldTable', + datasetQualityFlyoutDegradedTableNoData: 'datasetQualityFlyoutDegradedTableNoData', datasetQualityHeaderButton: 'datasetQualityHeaderButton', datasetQualityFlyoutFieldValue: 'datasetQualityFlyoutFieldValue', datasetQualityFlyoutIntegrationActionsButton: 'datasetQualityFlyoutIntegrationActionsButton', @@ -121,6 +138,10 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv await find.waitForDeletedByCssSelector('.euiBasicTable-loading', 20 * 1000); }, + async waitUntilTableInFlyoutLoaded() { + await find.waitForDeletedByCssSelector('.euiFlyoutBody .euiBasicTable-loading', 20 * 1000); + }, + async waitUntilSummaryPanelLoaded() { await testSubjects.missingOrFail(`datasetQuality-${texts.activeDatasets}-loading`); await testSubjects.missingOrFail(`datasetQuality-${texts.estimatedData}-loading`); @@ -159,6 +180,19 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv return testSubjects.find(testSubjectSelectors.datasetQualityTable); }, + getDatasetQualityFlyoutDegradedFieldTable(): Promise { + return testSubjects.find(testSubjectSelectors.datasetQualityFlyoutDegradedFieldTable); + }, + + async getDatasetQualityFlyoutDegradedFieldTableRows(): Promise { + await this.waitUntilTableInFlyoutLoaded(); + const table = await testSubjects.find( + testSubjectSelectors.datasetQualityFlyoutDegradedFieldTable + ); + const tBody = await table.findByTagName('tbody'); + return tBody.findAllByTagName('tr'); + }, + async refreshTable() { const filtersContainer = await testSubjects.find( testSubjectSelectors.datasetQualityFiltersContainer @@ -190,6 +224,11 @@ export function DatasetQualityPageObject({ getPageObjects, getService }: FtrProv ]); }, + async parseDegradedFieldTable() { + const table = await this.getDatasetQualityFlyoutDegradedFieldTable(); + return parseDatasetTable(table, ['Field', 'Count', 'Last Occurrence']); + }, + async filterForIntegrations(integrations: string[]) { return euiSelectable.selectOnlyOptionsWithText( testSubjectSelectors.datasetQualityIntegrationsSelectableButton, diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts index 68d96070990f0..399030d1dd377 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/data/logs_data.ts @@ -139,6 +139,56 @@ export function createLogRecord( .timestamp(timestamp); } +/* +The helped function generates 2 sets of Malformed Docs for the given dataset. +1 set has more Malformed fields than the second one. This help in having different counts and hence sorting + */ +export function createDegradedFieldsRecord({ + to, + count = 1, + dataset, +}: { + to: string; + count?: number; + dataset: string; +}) { + return timerange(moment(to).subtract(count, 'minute'), moment(to)) + .interval('1m') + .rate(1) + .generator((timestamp) => { + return Array(count) + .fill(0) + .flatMap((_, index) => [ + log + .create() + .dataset(dataset) + .message(MESSAGE_LOG_LEVELS[0].message) + .logLevel(MORE_THAN_1024_CHARS) + .service(SERVICE_NAMES[0]) + .namespace(defaultNamespace) + .defaults({ + 'trace.id': generateShortId(), + 'agent.name': 'synth-agent', + 'cloud.availability_zone': MORE_THAN_1024_CHARS, + }) + .timestamp(timestamp), + log + .create() + .dataset(dataset) + .message(MESSAGE_LOG_LEVELS[1].message) + .logLevel(MESSAGE_LOG_LEVELS[1].level) + .service(SERVICE_NAMES[0]) + .namespace(defaultNamespace) + .defaults({ + 'trace.id': generateShortId(), + 'agent.name': 'synth-agent', + 'cloud.availability_zone': MORE_THAN_1024_CHARS, + }) + .timestamp(timestamp), + ]); + }); +} + export const datasetNames = ['synth.1', 'synth.2', 'synth.3']; export const defaultNamespace = 'default'; diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts index 2f2c40932ceaf..7dc64b03335cb 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts @@ -7,7 +7,12 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; -import { datasetNames, getInitialTestLogs, getLogsForDataset } from './data'; +import { + datasetNames, + getInitialTestLogs, + getLogsForDataset, + createDegradedFieldsRecord, +} from './data'; const integrationActions = { overview: 'Overview', @@ -31,429 +36,537 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const to = '2024-01-01T12:00:00.000Z'; const excludeKeysFromServerless = ['size']; // https://github.com/elastic/kibana/issues/178954 - // FLAKY: https://github.com/elastic/kibana/issues/183771 - describe.skip('Dataset quality flyout', function () { - this.tags(['failsOnMKI']); // Failing https://github.com/elastic/kibana/issues/183495 + describe('Dataset quality flyout', function () { + // FLAKY: https://github.com/elastic/kibana/issues/183771 + // Added this sub describe block so that the existing flaky tests can be skipped and new ones can be added in the other describe block - before(async () => { - await PageObjects.svlCommonPage.loginWithRole('admin'); - await synthtrace.index(getInitialTestLogs({ to, count: 4 })); - await PageObjects.datasetQuality.navigateTo(); - }); + describe.skip('Other dataset quality flyout tests', () => { + this.tags(['failsOnMKI']); // Failing https://github.com/elastic/kibana/issues/183495 - after(async () => { - await synthtrace.clean(); - await PageObjects.observabilityLogsExplorer.removeInstalledPackages(); - }); + before(async () => { + await PageObjects.svlCommonPage.loginWithRole('admin'); + await synthtrace.index(getInitialTestLogs({ to, count: 4 })); + await PageObjects.datasetQuality.navigateTo(); + }); - it('opens the flyout for the right dataset', async () => { - const testDatasetName = datasetNames[1]; + after(async () => { + await synthtrace.clean(); + await PageObjects.observabilityLogsExplorer.removeInstalledPackages(); + }); - await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); + it('opens the flyout for the right dataset', async () => { + const testDatasetName = datasetNames[1]; - await testSubjects.existOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutTitle - ); - }); + await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); - // Fails on Serverless. TODO: Need to update the UI as well as the test - it.skip('shows the correct last activity', async () => { - const testDatasetName = datasetNames[0]; + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutTitle + ); + }); - // Update last activity for the dataset - await PageObjects.datasetQuality.closeFlyout(); - await synthtrace.index( - getLogsForDataset({ to: new Date().toISOString(), count: 1, dataset: testDatasetName }) - ); - await PageObjects.datasetQuality.refreshTable(); + // Fails on Serverless. TODO: Need to update the UI as well as the test + it.skip('shows the correct last activity', async () => { + const testDatasetName = datasetNames[0]; - const cols = await PageObjects.datasetQuality.parseDatasetTable(); + // Update last activity for the dataset + await PageObjects.datasetQuality.closeFlyout(); + await synthtrace.index( + getLogsForDataset({ to: new Date().toISOString(), count: 1, dataset: testDatasetName }) + ); + await PageObjects.datasetQuality.refreshTable(); - const datasetNameCol = cols['Dataset Name']; - const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); + const cols = await PageObjects.datasetQuality.parseDatasetTable(); - const testDatasetRowIndex = datasetNameColCellTexts.findIndex( - (dName: string) => dName === testDatasetName - ); + const datasetNameCol = cols['Dataset Name']; + const datasetNameColCellTexts = await datasetNameCol.getCellTexts(); - const lastActivityText = (await cols['Last Activity'].getCellTexts())[testDatasetRowIndex]; + const testDatasetRowIndex = datasetNameColCellTexts.findIndex( + (dName: string) => dName === testDatasetName + ); - await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); + const lastActivityText = (await cols['Last Activity'].getCellTexts())[testDatasetRowIndex]; - const lastActivityTextExists = await PageObjects.datasetQuality.doestTextExistInFlyout( - lastActivityText, - `[data-test-subj=${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutFieldValue}]` - ); + await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); - expect(lastActivityTextExists).to.eql(true); - }); + const lastActivityTextExists = await PageObjects.datasetQuality.doestTextExistInFlyout( + lastActivityText, + `[data-test-subj=${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutFieldValue}]` + ); - // FLAKY: https://github.com/elastic/kibana/issues/180994 - it.skip('reflects the breakdown field state in url', async () => { - const testDatasetName = datasetNames[0]; - await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); + expect(lastActivityTextExists).to.eql(true); + }); - const breakdownField = 'service.name'; - await PageObjects.datasetQuality.selectBreakdownField(breakdownField); + // FLAKY: https://github.com/elastic/kibana/issues/180994 + it.skip('reflects the breakdown field state in url', async () => { + const testDatasetName = datasetNames[0]; + await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); - // Wait for URL to contain "breakdownField:service.name" - await retry.tryForTime(5000, async () => { - const currentUrl = await browser.getCurrentUrl(); - expect(decodeURIComponent(currentUrl)).to.contain(`breakdownField:${breakdownField}`); - }); + const breakdownField = 'service.name'; + await PageObjects.datasetQuality.selectBreakdownField(breakdownField); - // Clear breakdown field - await PageObjects.datasetQuality.selectBreakdownField('No breakdown'); + // Wait for URL to contain "breakdownField:service.name" + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + expect(decodeURIComponent(currentUrl)).to.contain(`breakdownField:${breakdownField}`); + }); - // Wait for URL to not contain "breakdownField" - await retry.tryForTime(5000, async () => { - const currentUrl = await browser.getCurrentUrl(); - expect(currentUrl).to.not.contain('breakdownField'); + // Clear breakdown field + await PageObjects.datasetQuality.selectBreakdownField('No breakdown'); + + // Wait for URL to not contain "breakdownField" + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + expect(currentUrl).to.not.contain('breakdownField'); + }); }); - }); - it('shows the integration details', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; - const apacheIntegrationId = 'apache'; + it('shows the integration details', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + const apacheIntegrationId = 'apache'; - await PageObjects.observabilityLogsExplorer.navigateTo(); + await PageObjects.observabilityLogsExplorer.navigateTo(); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + // Add initial integrations + await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); - // Index 10 logs for `logs-apache.access` dataset - await synthtrace.index( - getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) - ); + // Index 10 logs for `logs-apache.access` dataset + await synthtrace.index( + getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) + ); - await PageObjects.datasetQuality.navigateTo(); + await PageObjects.datasetQuality.navigateTo(); - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - const integrationNameElements = await PageObjects.datasetQuality.getFlyoutElementsByText( - '[data-test-subj=datasetQualityFlyoutFieldValue]', - apacheIntegrationId - ); + const integrationNameElements = await PageObjects.datasetQuality.getFlyoutElementsByText( + '[data-test-subj=datasetQualityFlyoutFieldValue]', + apacheIntegrationId + ); - await PageObjects.datasetQuality.closeFlyout(); + await PageObjects.datasetQuality.closeFlyout(); - expect(integrationNameElements.length).to.eql(1); - }); + expect(integrationNameElements.length).to.eql(1); + }); - it('goes to log explorer page when open button is clicked', async () => { - const testDatasetName = datasetNames[2]; - await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); + it('goes to log explorer page when open button is clicked', async () => { + const testDatasetName = datasetNames[2]; + await PageObjects.datasetQuality.openDatasetFlyout(testDatasetName); - await (await PageObjects.datasetQuality.getFlyoutLogsExplorerButton()).click(); + await (await PageObjects.datasetQuality.getFlyoutLogsExplorerButton()).click(); - // Confirm dataset selector text in observability logs explorer - const datasetSelectorText = - await PageObjects.observabilityLogsExplorer.getDataSourceSelectorButtonText(); - expect(datasetSelectorText).to.eql(testDatasetName); - }); + // Confirm dataset selector text in observability logs explorer + const datasetSelectorText = + await PageObjects.observabilityLogsExplorer.getDataSourceSelectorButtonText(); + expect(datasetSelectorText).to.eql(testDatasetName); + }); - it('shows summary KPIs', async () => { - await PageObjects.datasetQuality.navigateTo(); + it('shows summary KPIs', async () => { + await PageObjects.datasetQuality.navigateTo(); - const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - const summary = await PageObjects.datasetQuality.parseFlyoutKpis(excludeKeysFromServerless); - expect(summary).to.eql({ - docsCountTotal: '0', - // size: '0.0 B', // `_stats` not available on Serverless - services: '0', - hosts: '0', - degradedDocs: '0', + const summary = await PageObjects.datasetQuality.parseFlyoutKpis(excludeKeysFromServerless); + expect(summary).to.eql({ + docsCountTotal: '0', + // size: '0.0 B', // `_stats` not available on Serverless + services: '0', + hosts: '0', + degradedDocs: '0', + }); }); - }); - it('shows the updated KPIs', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - - const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis( - excludeKeysFromServerless - ); - - // Set time range to 3 days ago - const flyoutBodyContainer = await testSubjects.find( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutBody - ); - await PageObjects.datasetQuality.setDatePickerLastXUnits(flyoutBodyContainer, 3, 'd'); - - // Index 2 doc 2 days ago - const time2DaysAgo = Date.now() - 2 * 24 * 60 * 60 * 1000; - await synthtrace.index( - getLogsForDataset({ - to: time2DaysAgo, - count: 2, - dataset: apacheAccessDatasetName, - isMalformed: false, - }) - ); - - // Index 5 degraded docs 2 days ago - await synthtrace.index( - getLogsForDataset({ - to: time2DaysAgo, - count: 5, - dataset: apacheAccessDatasetName, - isMalformed: true, - }) - ); - - await PageObjects.datasetQuality.refreshFlyout(); - const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis( - excludeKeysFromServerless - ); - - expect(parseInt(summaryAfter.docsCountTotal, 10)).to.be.greaterThan( - parseInt(summaryBefore.docsCountTotal, 10) - ); - - expect(parseInt(summaryAfter.degradedDocs, 10)).to.be.greaterThan( - parseInt(summaryBefore.degradedDocs, 10) - ); - - // `_stats` not available on Serverless so we can't compare size // https://github.com/elastic/kibana/issues/178954 - // expect(parseInt(summaryAfter.size, 10)).to.be.greaterThan(parseInt(summaryBefore.size, 10)); - - expect(parseInt(summaryAfter.services, 10)).to.be.greaterThan( - parseInt(summaryBefore.services, 10) - ); - expect(parseInt(summaryAfter.hosts, 10)).to.be.greaterThan(parseInt(summaryBefore.hosts, 10)); - }); + it('shows the updated KPIs', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - it('shows the right number of services', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - - const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis( - excludeKeysFromServerless - ); - const testServices = ['test-srv-1', 'test-srv-2']; - - // Index 2 docs with different services - const timeNow = Date.now(); - await synthtrace.index( - getLogsForDataset({ - to: timeNow, - count: 2, - dataset: apacheAccessDatasetName, - isMalformed: false, - services: testServices, - }) - ); - - await PageObjects.datasetQuality.refreshFlyout(); - const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis( - excludeKeysFromServerless - ); - - expect(parseInt(summaryAfter.services, 10)).to.eql( - parseInt(summaryBefore.services, 10) + testServices.length - ); - }); + const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis( + excludeKeysFromServerless + ); - it('goes to log explorer for degraded docs when show all is clicked', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + // Set time range to 3 days ago + const flyoutBodyContainer = await testSubjects.find( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutBody + ); + await PageObjects.datasetQuality.setDatePickerLastXUnits(flyoutBodyContainer, 3, 'd'); + + // Index 2 doc 2 days ago + const time2DaysAgo = Date.now() - 2 * 24 * 60 * 60 * 1000; + await synthtrace.index( + getLogsForDataset({ + to: time2DaysAgo, + count: 2, + dataset: apacheAccessDatasetName, + isMalformed: false, + }) + ); - const degradedDocsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.degradedDocs}`; - await testSubjects.click(degradedDocsShowAllSelector); - await browser.switchTab(1); + // Index 5 degraded docs 2 days ago + await synthtrace.index( + getLogsForDataset({ + to: time2DaysAgo, + count: 5, + dataset: apacheAccessDatasetName, + isMalformed: true, + }) + ); - // Confirm dataset selector text in observability logs explorer - const datasetSelectorText = - await PageObjects.observabilityLogsExplorer.getDataSourceSelectorButtonText(); - expect(datasetSelectorText).to.contain(apacheAccessDatasetName); + await PageObjects.datasetQuality.refreshFlyout(); + const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis( + excludeKeysFromServerless + ); - await browser.closeCurrentWindow(); - await browser.switchTab(0); - }); + expect(parseInt(summaryAfter.docsCountTotal, 10)).to.be.greaterThan( + parseInt(summaryBefore.docsCountTotal, 10) + ); - // Blocked by https://github.com/elastic/kibana/issues/181705 - it.skip('goes to infra hosts for hosts when show all is clicked', async () => { - const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + expect(parseInt(summaryAfter.degradedDocs, 10)).to.be.greaterThan( + parseInt(summaryBefore.degradedDocs, 10) + ); - const hostsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.hosts}`; - await testSubjects.click(hostsShowAllSelector); - await browser.switchTab(1); + // `_stats` not available on Serverless so we can't compare size // https://github.com/elastic/kibana/issues/178954 + // expect(parseInt(summaryAfter.size, 10)).to.be.greaterThan(parseInt(summaryBefore.size, 10)); - // Confirm url contains metrics/hosts - await retry.tryForTime(5000, async () => { - const currentUrl = await browser.getCurrentUrl(); - const parsedUrl = new URL(currentUrl); - expect(parsedUrl.pathname).to.contain('/app/metrics/hosts'); + expect(parseInt(summaryAfter.services, 10)).to.be.greaterThan( + parseInt(summaryBefore.services, 10) + ); + expect(parseInt(summaryAfter.hosts, 10)).to.be.greaterThan( + parseInt(summaryBefore.hosts, 10) + ); }); - await browser.closeCurrentWindow(); - await browser.switchTab(0); - }); + it('shows the right number of services', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - it('Integration actions menu is present with correct actions', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; + const summaryBefore = await PageObjects.datasetQuality.parseFlyoutKpis( + excludeKeysFromServerless + ); + const testServices = ['test-srv-1', 'test-srv-2']; + + // Index 2 docs with different services + const timeNow = Date.now(); + await synthtrace.index( + getLogsForDataset({ + to: timeNow, + count: 2, + dataset: apacheAccessDatasetName, + isMalformed: false, + services: testServices, + }) + ); - await PageObjects.observabilityLogsExplorer.navigateTo(); + await PageObjects.datasetQuality.refreshFlyout(); + const summaryAfter = await PageObjects.datasetQuality.parseFlyoutKpis( + excludeKeysFromServerless + ); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + expect(parseInt(summaryAfter.services, 10)).to.eql( + parseInt(summaryBefore.services, 10) + testServices.length + ); + }); - // Index 10 logs for `logs-apache.access` dataset - await synthtrace.index( - getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) - ); + it('goes to log explorer for degraded docs when show all is clicked', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - await PageObjects.datasetQuality.navigateTo(); + const degradedDocsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.degradedDocs}`; + await testSubjects.click(degradedDocsShowAllSelector); + await browser.switchTab(1); - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - await PageObjects.datasetQuality.openIntegrationActionsMenu(); + // Confirm dataset selector text in observability logs explorer + const datasetSelectorText = + await PageObjects.observabilityLogsExplorer.getDataSourceSelectorButtonText(); + expect(datasetSelectorText).to.contain(apacheAccessDatasetName); - const actions = await Promise.all( - Object.values(integrationActions).map((action) => - PageObjects.datasetQuality.getIntegrationActionButtonByAction(action) - ) - ); + await browser.closeCurrentWindow(); + await browser.switchTab(0); + }); - expect(actions.length).to.eql(3); - }); + // Blocked by https://github.com/elastic/kibana/issues/181705 + it.skip('goes to infra hosts for hosts when show all is clicked', async () => { + const apacheAccessDatasetHumanName = 'Apache access logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - it('Integration dashboard action hidden for integrations without dashboards', async () => { - const bitbucketDatasetName = 'atlassian_bitbucket.audit'; - const bitbucketDatasetHumanName = 'Bitbucket Audit Logs'; + const hostsShowAllSelector = `${PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutKpiLink}-${PageObjects.datasetQuality.texts.hosts}`; + await testSubjects.click(hostsShowAllSelector); + await browser.switchTab(1); - await PageObjects.observabilityLogsExplorer.navigateTo(); + // Confirm url contains metrics/hosts + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + const parsedUrl = new URL(currentUrl); + expect(parsedUrl.pathname).to.contain('/app/metrics/hosts'); + }); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.installPackage({ - name: 'atlassian_bitbucket', - version: '1.14.0', + await browser.closeCurrentWindow(); + await browser.switchTab(0); }); - // Index 10 logs for `atlassian_bitbucket.audit` dataset - await synthtrace.index(getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName })); + it('Integration actions menu is present with correct actions', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; - await PageObjects.datasetQuality.navigateTo(); + await PageObjects.observabilityLogsExplorer.navigateTo(); - await PageObjects.datasetQuality.openDatasetFlyout(bitbucketDatasetHumanName); - await PageObjects.datasetQuality.openIntegrationActionsMenu(); + // Add initial integrations + await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); - await testSubjects.missingOrFail( - PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutIntegrationAction( - integrationActions.viewDashboards - ) - ); - }); + // Index 10 logs for `logs-apache.access` dataset + await synthtrace.index( + getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) + ); + + await PageObjects.datasetQuality.navigateTo(); - it('Integration overview action should navigate to the integration overview page', async () => { - const bitbucketDatasetName = 'atlassian_bitbucket.audit'; - const bitbucketDatasetHumanName = 'Bitbucket Audit Logs'; + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + await PageObjects.datasetQuality.openIntegrationActionsMenu(); - await PageObjects.observabilityLogsExplorer.navigateTo(); + const actions = await Promise.all( + Object.values(integrationActions).map((action) => + PageObjects.datasetQuality.getIntegrationActionButtonByAction(action) + ) + ); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.installPackage({ - name: 'atlassian_bitbucket', - version: '1.14.0', + expect(actions.length).to.eql(3); }); - // Index 10 logs for `atlassian_bitbucket.audit` dataset - await synthtrace.index(getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName })); + it('Integration dashboard action hidden for integrations without dashboards', async () => { + const bitbucketDatasetName = 'atlassian_bitbucket.audit'; + const bitbucketDatasetHumanName = 'Bitbucket Audit Logs'; - await PageObjects.datasetQuality.navigateTo(); + await PageObjects.observabilityLogsExplorer.navigateTo(); - await PageObjects.datasetQuality.openDatasetFlyout(bitbucketDatasetHumanName); - await PageObjects.datasetQuality.openIntegrationActionsMenu(); + // Add initial integrations + await PageObjects.observabilityLogsExplorer.installPackage({ + name: 'atlassian_bitbucket', + version: '1.14.0', + }); - const action = await PageObjects.datasetQuality.getIntegrationActionButtonByAction( - integrationActions.overview - ); + // Index 10 logs for `atlassian_bitbucket.audit` dataset + await synthtrace.index(getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName })); - await action.click(); + await PageObjects.datasetQuality.navigateTo(); - await retry.tryForTime(5000, async () => { - const currentUrl = await browser.getCurrentUrl(); - const parsedUrl = new URL(currentUrl); + await PageObjects.datasetQuality.openDatasetFlyout(bitbucketDatasetHumanName); + await PageObjects.datasetQuality.openIntegrationActionsMenu(); - expect(parsedUrl.pathname).to.contain('/app/integrations/detail/atlassian_bitbucket'); + await testSubjects.missingOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutIntegrationAction( + integrationActions.viewDashboards + ) + ); }); - }); - it('Integration template action should navigate to the index template page', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; + it('Integration overview action should navigate to the integration overview page', async () => { + const bitbucketDatasetName = 'atlassian_bitbucket.audit'; + const bitbucketDatasetHumanName = 'Bitbucket Audit Logs'; - await PageObjects.observabilityLogsExplorer.navigateTo(); + await PageObjects.observabilityLogsExplorer.navigateTo(); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + // Add initial integrations + await PageObjects.observabilityLogsExplorer.installPackage({ + name: 'atlassian_bitbucket', + version: '1.14.0', + }); - // Index 10 logs for `logs-apache.access` dataset - await synthtrace.index( - getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) - ); + // Index 10 logs for `atlassian_bitbucket.audit` dataset + await synthtrace.index(getLogsForDataset({ to, count: 10, dataset: bitbucketDatasetName })); - await PageObjects.datasetQuality.navigateTo(); + await PageObjects.datasetQuality.navigateTo(); - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - await PageObjects.datasetQuality.openIntegrationActionsMenu(); + await PageObjects.datasetQuality.openDatasetFlyout(bitbucketDatasetHumanName); + await PageObjects.datasetQuality.openIntegrationActionsMenu(); - await retry.tryForTime(5000, async () => { const action = await PageObjects.datasetQuality.getIntegrationActionButtonByAction( - integrationActions.template + integrationActions.overview ); await action.click(); - const currentUrl = await browser.getCurrentUrl(); - const parsedUrl = new URL(currentUrl); - expect(parsedUrl.pathname).to.contain( - `/app/management/data/index_management/templates/logs-${apacheAccessDatasetName}` + await retry.tryForTime(5000, async () => { + const currentUrl = await browser.getCurrentUrl(); + const parsedUrl = new URL(currentUrl); + + expect(parsedUrl.pathname).to.contain('/app/integrations/detail/atlassian_bitbucket'); + }); + }); + + it('Integration template action should navigate to the index template page', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + + await PageObjects.observabilityLogsExplorer.navigateTo(); + + // Add initial integrations + await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + + // Index 10 logs for `logs-apache.access` dataset + await synthtrace.index( + getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) ); + + await PageObjects.datasetQuality.navigateTo(); + + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + await PageObjects.datasetQuality.openIntegrationActionsMenu(); + + await retry.tryForTime(5000, async () => { + const action = await PageObjects.datasetQuality.getIntegrationActionButtonByAction( + integrationActions.template + ); + + await action.click(); + + const currentUrl = await browser.getCurrentUrl(); + const parsedUrl = new URL(currentUrl); + expect(parsedUrl.pathname).to.contain( + `/app/management/data/index_management/templates/logs-${apacheAccessDatasetName}` + ); + }); + }); + + it('Integration dashboard action should navigate to the selected dashboard', async () => { + const apacheAccessDatasetName = 'apache.access'; + const apacheAccessDatasetHumanName = 'Apache access logs'; + + await PageObjects.observabilityLogsExplorer.navigateTo(); + + // Add initial integrations + await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + + // Index 10 logs for `logs-apache.access` dataset + await synthtrace.index( + getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) + ); + + await PageObjects.datasetQuality.navigateTo(); + + await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); + await PageObjects.datasetQuality.openIntegrationActionsMenu(); + + const action = await PageObjects.datasetQuality.getIntegrationActionButtonByAction( + integrationActions.viewDashboards + ); + + await action.click(); + + const dashboardButtons = await PageObjects.datasetQuality.getIntegrationDashboardButtons(); + const firstDashboardButton = await dashboardButtons[0]; + const dashboardText = await firstDashboardButton.getVisibleText(); + + await firstDashboardButton.click(); + + const breadcrumbText = await testSubjects.getVisibleText('breadcrumb last'); + + expect(breadcrumbText).to.eql(dashboardText); }); }); - it('Integration dashboard action should navigate to the selected dashboard', async () => { - const apacheAccessDatasetName = 'apache.access'; - const apacheAccessDatasetHumanName = 'Apache access logs'; + // The above describe block has some failing/flaky tests which will + // be fixed as part of the tech debt mentioned here + // https://github.com/elastic/kibana/issues/184145 + // Until then, the below describe block is added to cover the tests for the + // newly added degraded Fields Table. This must be merged under the above + // describe block once the tech debt is fixed. + describe('Dataset quality flyout with degraded fields', () => { + const goodDatasetName = 'good'; + const degradedDatasetName = 'degraded'; + const today = Date.now(); + before(async () => { + await PageObjects.svlCommonPage.loginWithRole('admin'); + await synthtrace.index([ + getLogsForDataset({ + to: today, + count: 2, + dataset: goodDatasetName, + isMalformed: false, + }), + createDegradedFieldsRecord({ + to: today, + count: 2, + dataset: degradedDatasetName, + }), + ]); + await PageObjects.datasetQuality.navigateTo(); + }); + + after(async () => { + await synthtrace.clean(); + }); + + it('shows the degraded fields table with no data when no degraded fields are present', async () => { + await PageObjects.datasetQuality.openDatasetFlyout(goodDatasetName); + + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedTableNoData + ); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should load the degraded fields table with data', async () => { + await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName); - await PageObjects.observabilityLogsExplorer.navigateTo(); + await testSubjects.existOrFail( + PageObjects.datasetQuality.testSubjectSelectors.datasetQualityFlyoutDegradedFieldTable + ); + + const rows = + await PageObjects.datasetQuality.getDatasetQualityFlyoutDegradedFieldTableRows(); + + expect(rows.length).to.eql(2); + + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should sort the table when the count table header is clicked', async () => { + await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName); - // Add initial integrations - await PageObjects.observabilityLogsExplorer.setupInitialIntegrations(); + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); - // Index 10 logs for `logs-apache.access` dataset - await synthtrace.index( - getLogsForDataset({ to, count: 10, dataset: apacheAccessDatasetName }) - ); + const countColumn = table.Count; + const cellTexts = await countColumn.getCellTexts(); - await PageObjects.datasetQuality.navigateTo(); + await countColumn.sort('ascending'); + const sortedCellTexts = await countColumn.getCellTexts(); - await PageObjects.datasetQuality.openDatasetFlyout(apacheAccessDatasetHumanName); - await PageObjects.datasetQuality.openIntegrationActionsMenu(); + expect(cellTexts.reverse()).to.eql(sortedCellTexts); - const action = await PageObjects.datasetQuality.getIntegrationActionButtonByAction( - integrationActions.viewDashboards - ); + await PageObjects.datasetQuality.closeFlyout(); + }); + + it('should update the table when new data is ingested and the flyout is refreshed using the time selector', async () => { + await PageObjects.datasetQuality.openDatasetFlyout(degradedDatasetName); + + const table = await PageObjects.datasetQuality.parseDegradedFieldTable(); - await action.click(); + const countColumn = table.Count; + const cellTexts = await countColumn.getCellTexts(); - const dashboardButtons = await PageObjects.datasetQuality.getIntegrationDashboardButtons(); - const firstDashboardButton = await dashboardButtons[0]; - const dashboardText = await firstDashboardButton.getVisibleText(); + await synthtrace.index([ + createDegradedFieldsRecord({ + to: today, + count: 2, + dataset: degradedDatasetName, + }), + ]); - await firstDashboardButton.click(); + await PageObjects.datasetQuality.refreshFlyout(); - const breadcrumbText = await testSubjects.getVisibleText('breadcrumb last'); + const updatedCellTexts = await countColumn.getCellTexts(); - expect(breadcrumbText).to.eql(dashboardText); + const singleValuePreviously = parseInt(cellTexts[0], 10); + const singleValueNow = parseInt(updatedCellTexts[0], 10); + + expect(singleValueNow).to.be(singleValuePreviously * 2); + + await PageObjects.datasetQuality.closeFlyout(); + }); }); }); } From 00563cd74feca370e48489fe8c8cb53fb710381c Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Mon, 27 May 2024 12:08:30 +0200 Subject: [PATCH 07/11] Fix checktype issue on Public State --- .../dataset_quality/public/controller/public_state.ts | 4 ++++ .../dataset_quality/public/controller/types.ts | 5 +---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts b/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts index 22a37a60b6cc2..64eff0d0fd222 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts @@ -39,6 +39,10 @@ export const getContextFromPublicState = ( flyout: { ...DEFAULT_CONTEXT.flyout, ...publicState.flyout, + degradedFields: { + table: + publicState.flyout?.degradedFields?.table ?? DEFAULT_CONTEXT.flyout.degradedFields.table, + }, }, filters: { ...DEFAULT_CONTEXT.filters, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts index a3e211432b61e..a74b5ed498717 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts @@ -24,10 +24,7 @@ export type DatasetQualityTableOptions = Partial< Omit & { sort: TableSortOptions } >; -export type DatasetQualityFlyoutOptions = Omit< - WithFlyoutOptions['flyout'], - 'datasetDetails' | 'degradedFields' -> & { degradedFields: { table: DatasetQualityTableOptions } }; +export type DatasetQualityFlyoutOptions = Omit; export type DatasetQualityFilterOptions = Partial; From 803f0678b8a9d6f0315fa5b6fa032f04949c6e02 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Mon, 27 May 2024 16:43:49 +0200 Subject: [PATCH 08/11] Fix type issue with url schema --- .../public/controller/public_state.ts | 14 +++++++++++--- .../dataset_quality/public/controller/types.ts | 15 ++++++++++++++- .../dataset_quality_controller/src/types.ts | 2 +- .../url_schema/dataset_quality/url_schema_v1.ts | 8 +++++--- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts b/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts index 64eff0d0fd222..9500b473c95ed 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/controller/public_state.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { DatasetTableSortField } from '../hooks'; +import { DatasetTableSortField, DegradedFieldSortField } from '../hooks'; import { DatasetQualityControllerContext, DEFAULT_CONTEXT, @@ -40,8 +40,16 @@ export const getContextFromPublicState = ( ...DEFAULT_CONTEXT.flyout, ...publicState.flyout, degradedFields: { - table: - publicState.flyout?.degradedFields?.table ?? DEFAULT_CONTEXT.flyout.degradedFields.table, + table: { + ...DEFAULT_CONTEXT.flyout.degradedFields.table, + ...publicState.flyout?.degradedFields?.table, + sort: publicState.flyout?.degradedFields?.table?.sort + ? { + ...publicState.flyout.degradedFields.table.sort, + field: publicState.flyout.degradedFields.table.sort.field as DegradedFieldSortField, + } + : DEFAULT_CONTEXT.flyout.degradedFields.table.sort, + }, }, }, filters: { diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts index a74b5ed498717..66757d3409567 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/controller/types.ts @@ -11,6 +11,7 @@ import { WithFilters, WithFlyoutOptions, WithTableOptions, + DegradedFields, } from '../state_machines/dataset_quality_controller'; export interface DatasetQualityController { @@ -24,7 +25,19 @@ export type DatasetQualityTableOptions = Partial< Omit & { sort: TableSortOptions } >; -export type DatasetQualityFlyoutOptions = Omit; +type DegradedFieldSortOptions = Omit & { field: string }; + +export type DatasetQualityDegradedFieldTableOptions = Partial< + Omit & { + sort: DegradedFieldSortOptions; + } +>; + +export type DatasetQualityFlyoutOptions = Partial< + Omit & { + degradedFields: { table?: DatasetQualityDegradedFieldTableOptions }; + } +>; export type DatasetQualityFilterOptions = Partial; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index 2e995c6a50590..2dc1ba19d3c52 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -38,7 +38,7 @@ interface TableCriteria { }; } -interface DegradedFields { +export interface DegradedFields { table: TableCriteria; data?: DegradedField[]; } diff --git a/x-pack/plugins/observability_solution/observability_logs_explorer/common/url_schema/dataset_quality/url_schema_v1.ts b/x-pack/plugins/observability_solution/observability_logs_explorer/common/url_schema/dataset_quality/url_schema_v1.ts index c97210b2a194b..6fd5781a217e8 100644 --- a/x-pack/plugins/observability_solution/observability_logs_explorer/common/url_schema/dataset_quality/url_schema_v1.ts +++ b/x-pack/plugins/observability_solution/observability_logs_explorer/common/url_schema/dataset_quality/url_schema_v1.ts @@ -55,9 +55,11 @@ const timeRangeRT = rt.strict({ }), }); -const degradedFieldRT = rt.strict({ - table: tableRT, -}); +const degradedFieldRT = rt.exact( + rt.partial({ + table: tableRT, + }) +); export const flyoutRT = rt.exact( rt.partial({ From 75113c7ca2bf7328e3286bb04147cb9f99a85858 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Tue, 28 May 2024 10:38:29 +0200 Subject: [PATCH 09/11] Fix ts issue with synthtrace --- .../functional/apps/dataset_quality/dataset_quality_flyout.ts | 2 +- .../observability/dataset_quality/dataset_quality_flyout.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts b/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts index 5a3902aafc7aa..3bd17373123ee 100644 --- a/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts +++ b/x-pack/test/functional/apps/dataset_quality/dataset_quality_flyout.ts @@ -457,7 +457,7 @@ export default function ({ getService, getPageObjects }: DatasetQualityFtrProvid describe('Dataset quality flyout with degraded fields', () => { const goodDatasetName = 'good'; const degradedDatasetName = 'degraded'; - const today = Date.now(); + const today = new Date().toISOString(); before(async () => { await synthtrace.index([ getLogsForDataset({ diff --git a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts index 7dc64b03335cb..a79c93f922a33 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_flyout.ts @@ -476,7 +476,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('Dataset quality flyout with degraded fields', () => { const goodDatasetName = 'good'; const degradedDatasetName = 'degraded'; - const today = Date.now(); + const today = new Date().toISOString(); before(async () => { await PageObjects.svlCommonPage.loginWithRole('admin'); await synthtrace.index([ From e24a79c11a16f8496d4759d5696bffaf847bffd1 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Tue, 28 May 2024 16:42:06 +0200 Subject: [PATCH 10/11] Fix review comments iteration 1 --- .../dataset_quality/common/api_types.ts | 14 ++++++---- .../common/data_streams_stats/types.ts | 9 ++++-- .../flyout/degraded_fields/columns.tsx | 21 +++++++------- .../degraded_fields/degraded_fields.tsx | 28 +++++-------------- .../flyout/degraded_fields/table.tsx | 12 +++++--- .../use_dataset_quality_degraded_field.ts | 17 ++++++----- .../data_stream_details_client.ts | 4 +-- .../services/data_stream_details/types.ts | 6 ++-- .../src/state_machine.ts | 8 ++---- .../dataset_quality_controller/src/types.ts | 3 +- .../data_streams/get_degraded_fields/index.ts | 21 +++++++------- .../server/routes/data_streams/routes.ts | 4 +-- .../data_streams/degraded_fields.spec.ts | 6 ++-- 13 files changed, 78 insertions(+), 75 deletions(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts index e2a918de285f1..0827329f4ceb8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts @@ -79,15 +79,19 @@ export const degradedDocsRt = rt.type({ export type DegradedDocs = rt.TypeOf; export const degradedFieldRt = rt.type({ - fieldName: rt.string, + name: rt.string, count: rt.number, - last_occurrence: rt.union([rt.null, rt.number]), + lastOccurrence: rt.union([rt.null, rt.number]), }); -export const getDataStreamDegradedFieldsResponseRt = rt.array(degradedFieldRt); - export type DegradedField = rt.TypeOf; +export const getDataStreamDegradedFieldsResponseRt = rt.type({ + degradedFields: rt.array(degradedFieldRt), +}); + +export type DegradedFieldResponse = rt.TypeOf; + export const dataStreamSettingsRt = rt.partial({ createdOn: rt.union([rt.null, rt.number]), // rt.null is needed because `createdOn` is not available on Serverless }); @@ -125,8 +129,6 @@ export const dataStreamsEstimatedDataInBytesRT = rt.type({ estimatedDataInBytes: rt.union([rt.number, rt.null]), // Null in serverless: https://github.com/elastic/kibana/issues/178954 }); -export type DataStreamsEstimatedDataInBytes = rt.TypeOf; - export const getDataStreamsEstimatedDataInBytesResponseRt = rt.exact( dataStreamsEstimatedDataInBytesRT ); diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts index 9bb8255b08032..b2426ef53aeb5 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts @@ -26,8 +26,8 @@ export type GetDataStreamsDegradedDocsStatsParams = export type GetDataStreamsDegradedDocsStatsQuery = GetDataStreamsDegradedDocsStatsParams['query']; export type GetDataStreamsDegradedDocsStatsResponse = APIReturnType<`GET /internal/dataset_quality/data_streams/degraded_docs`>; -export type DataStreamDegradedDocsStatServiceResponse = DegradedDocsStatType[]; export type DegradedDocsStatType = GetDataStreamsDegradedDocsStatsResponse['degradedDocs'][0]; +export type DataStreamDegradedDocsStatServiceResponse = DegradedDocsStatType[]; /* Types for Degraded Fields inside a DataStream @@ -74,4 +74,9 @@ export type GetIntegrationDashboardsResponse = export type DashboardType = GetIntegrationDashboardsResponse['dashboards'][0]; export type { DataStreamStat } from './data_stream_stat'; -export type { DataStreamDetails, DataStreamSettings, DegradedField } from '../api_types'; +export type { + DataStreamDetails, + DataStreamSettings, + DegradedField, + DegradedFieldResponse, +} from '../api_types'; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/columns.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/columns.tsx index 7cc96aa665966..10b30fe855aa2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/columns.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/columns.tsx @@ -6,7 +6,9 @@ */ import { EuiBasicTableColumn } from '@elastic/eui'; +import { FieldFormat } from '@kbn/field-formats-plugin/common'; import { i18n } from '@kbn/i18n'; + import { DegradedField } from '../../../../common/api_types'; const fieldColumnName = i18n.translate('xpack.datasetQuality.flyout.degradedField.field', { @@ -18,16 +20,20 @@ const countColumnName = i18n.translate('xpack.datasetQuality.flyout.degradedFiel }); const lastOccurrenceColumnName = i18n.translate( - 'xpack.datasetQuality.flyout.degradedField.last_occurrence', + 'xpack.datasetQuality.flyout.degradedField.lastOccurrence', { defaultMessage: 'Last Occurrence', } ); -export const getDegradedFieldsColumns = (): Array> => [ +export const getDegradedFieldsColumns = ({ + dateFormatter, +}: { + dateFormatter: FieldFormat; +}): Array> => [ { name: fieldColumnName, - field: 'fieldName', + field: 'name', }, { name: countColumnName, @@ -38,14 +44,9 @@ export const getDegradedFieldsColumns = (): Array { - if (lastOccurrence) { - const date = new Date(lastOccurrence); - return date.toISOString(); - } - - return ''; + return dateFormatter.convert(lastOccurrence); }, }, ]; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/degraded_fields.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/degraded_fields.tsx index 9995573725a82..bd2c20b24171a 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/degraded_fields.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/degraded_fields.tsx @@ -6,34 +6,20 @@ */ import React from 'react'; -import { EuiFlexGroup, EuiFlexItem, EuiIcon, EuiPanel, EuiTitle, EuiToolTip } from '@elastic/eui'; -import { css } from '@emotion/react'; +import { EuiFlexGroup, EuiPanel, EuiTitle, EuiIconTip } from '@elastic/eui'; import { flyoutImprovementText, flyoutImprovementTooltip } from '../../../../common/translations'; import { DegradedFieldTable } from './table'; export function DegradedFields() { return ( - - - -
{flyoutImprovementText}
- - - - - - - - + + +
{flyoutImprovementText}
+ + + ); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx index 419095865e5d1..b14bc6dbf57f2 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx +++ b/x-pack/plugins/observability_solution/dataset_quality/public/components/flyout/degraded_fields/table.tsx @@ -7,6 +7,7 @@ import { EuiBasicTable, EuiEmptyPrompt } from '@elastic/eui'; import React from 'react'; +import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '@kbn/field-types'; import { useDatasetQualityDegradedField } from '../../../hooks'; import { getDegradedFieldsColumns } from './columns'; import { @@ -15,16 +16,19 @@ import { } from '../../../../common/translations'; export const DegradedFieldTable = () => { - const { loadingState, pagination, renderedItems, onTableChange, sort } = + const { isLoading, pagination, renderedItems, onTableChange, sort, fieldFormats } = useDatasetQualityDegradedField(); - const columns = getDegradedFieldsColumns(); + const dateFormatter = fieldFormats.getDefaultInstance(KBN_FIELD_TYPES.DATE, [ + ES_FIELD_TYPES.DATE, + ]); + const columns = getDegradedFieldsColumns({ dateFormatter }); return ( { 'data-test-subj': 'datasetQualityFlyoutDegradedTableRow', }} noItemsMessage={ - loadingState.datasetDegradedFieldsLoading ? ( + isLoading ? ( flyoutDegradedFieldsTableLoadingText ) : ( state.context.flyout) ?? {}; + const degradedFields = useSelector(service, (state) => state.context.flyout.degradedFields) ?? {}; const { data, table } = degradedFields; const { page, rowsPerPage, sort } = table; @@ -56,17 +60,16 @@ export function useDatasetQualityDegradedField() { return sortedItems.slice(page * rowsPerPage, (page + 1) * rowsPerPage); }, [data, sort.field, sort.direction, page, rowsPerPage]); - const loadingState = useSelector(service, (state) => ({ - datasetDegradedFieldsLoading: state.matches( - 'flyout.initializing.dataStreamDegradedFields.fetching' - ), - })); + const isLoading = useSelector(service, (state) => + state.matches('flyout.initializing.dataStreamDegradedFields.fetching') + ); return { - loadingState, + isLoading, pagination, onTableChange, renderedItems, sort: { sort }, + fieldFormats, }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts index 38d396e83386a..e1282eb494999 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/data_stream_details_client.ts @@ -8,7 +8,6 @@ import { HttpStart } from '@kbn/core/public'; import { decodeOrThrow } from '@kbn/io-ts-utils'; import { - DegradedField, getDataStreamDegradedFieldsResponseRt, getDataStreamsDetailsResponseRt, getDataStreamsSettingsResponseRt, @@ -17,6 +16,7 @@ import { import { DataStreamDetails, DataStreamSettings, + DegradedFieldResponse, GetDataStreamDegradedFieldsParams, GetDataStreamDetailsParams, GetDataStreamDetailsResponse, @@ -77,7 +77,7 @@ export class DataStreamDetailsClient implements IDataStreamDetailsClient { end, }: GetDataStreamDegradedFieldsParams) { const response = await this.http - .get( + .get( `/internal/dataset_quality/data_streams/${dataStream}/degraded_fields`, { query: { start, end }, diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts index 807b79a1d87fe..708c1ba8a6c4c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/services/data_stream_details/types.ts @@ -14,7 +14,7 @@ import { GetIntegrationDashboardsParams, GetIntegrationDashboardsResponse, GetDataStreamDegradedFieldsParams, - DegradedField, + DegradedFieldResponse, } from '../../../common/data_streams_stats'; export type DataStreamDetailsServiceSetup = void; @@ -30,7 +30,9 @@ export interface DataStreamDetailsServiceStartDeps { export interface IDataStreamDetailsClient { getDataStreamSettings(params: GetDataStreamSettingsParams): Promise; getDataStreamDetails(params: GetDataStreamDetailsParams): Promise; - getDataStreamDegradedFields(params: GetDataStreamDegradedFieldsParams): Promise; + getDataStreamDegradedFields( + params: GetDataStreamDegradedFieldsParams + ): Promise; getIntegrationDashboards( params: GetIntegrationDashboardsParams ): Promise; diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts index 802a5b8e7d66a..2da9fe409aee8 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/state_machine.ts @@ -8,7 +8,7 @@ import { IToasts } from '@kbn/core/public'; import { getDateISORange } from '@kbn/timerange'; import { assign, createMachine, DoneInvokeEvent, InterpreterFrom } from 'xstate'; -import { DataStreamStat, DegradedField } from '../../../../common/api_types'; +import { DataStreamStat, DegradedFieldResponse } from '../../../../common/api_types'; import { Integration } from '../../../../common/data_streams_stats/integration'; import { IDataStreamDetailsClient } from '../../../services/data_stream_details'; import { @@ -532,14 +532,14 @@ export const createPureDatasetQualityControllerStateMachine = ( } : {}; }), - storeDegradedFields: assign((context, event) => { + storeDegradedFields: assign((context, event: DoneInvokeEvent) => { return 'data' in event ? { flyout: { ...context.flyout, degradedFields: { ...context.flyout.degradedFields, - data: event.data as DegradedField[], + data: event.data.degradedFields, }, }, } @@ -684,8 +684,6 @@ export const createDatasetQualityControllerStateMachine = ({ loadDegradedFieldsPerDataStream: (context) => { if (!context.flyout.dataset || !context.flyout.insightsTimeRange) { - fetchDatasetDetailsFailedNotifier(toasts, new Error(noDatasetSelected)); - return Promise.resolve({}); } diff --git a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts index 2dc1ba19d3c52..286f621fe432c 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/types.ts @@ -22,6 +22,7 @@ import { DataStreamStatType, GetNonAggregatableDataStreamsResponse, DegradedField, + DegradedFieldResponse, } from '../../../../common/data_streams_stats'; export type FlyoutDataset = Omit< @@ -219,7 +220,7 @@ export type DatasetQualityControllerEvent = | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent - | DoneInvokeEvent + | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent | DoneInvokeEvent diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts index 6bc0eb6a78a0a..92a1565afb9bc 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_degraded_fields/index.ts @@ -7,7 +7,7 @@ import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; import { rangeQuery, existsQuery } from '@kbn/observability-plugin/server'; -import { DegradedField } from '../../../../common/api_types'; +import { DegradedFieldResponse } from '../../../../common/api_types'; import { MAX_DEGRADED_FIELDS } from '../../../../common/constants'; import { createDatasetQualityESClient } from '../../../utils'; import { _IGNORED, TIMESTAMP } from '../../../../common/es_fields'; @@ -22,7 +22,7 @@ export async function getDegradedFields({ start: number; end: number; dataStream: string; -}): Promise { +}): Promise { const datasetQualityESClient = createDatasetQualityESClient(esClient); const filterQuery = [...rangeQuery(start, end)]; @@ -36,7 +36,7 @@ export async function getDegradedFields({ field: _IGNORED, }, aggs: { - last_occurrence: { + lastOccurrence: { max: { field: TIMESTAMP, }, @@ -57,11 +57,12 @@ export async function getDegradedFields({ aggs, }); - return ( - response.aggregations?.degradedFields.buckets.map((bucket) => ({ - fieldName: bucket.key as string, - count: bucket.doc_count, - last_occurrence: bucket.last_occurrence.value, - })) ?? [] - ); + return { + degradedFields: + response.aggregations?.degradedFields.buckets.map((bucket) => ({ + name: bucket.key as string, + count: bucket.doc_count, + lastOccurrence: bucket.lastOccurrence.value, + })) ?? [], + }; } diff --git a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts index 92e8b621f8aa0..26761e53575fd 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts @@ -13,7 +13,7 @@ import { DataStreamStat, DegradedDocs, NonAggregatableDatasets, - DegradedField, + DegradedFieldResponse, } from '../../../common/api_types'; import { indexNameToDataStreamParts } from '../../../common/utils'; import { rangeRt, typeRt } from '../../types/default_api_types'; @@ -136,7 +136,7 @@ const degradedFieldsRoute = createDatasetQualityServerRoute({ options: { tags: [], }, - async handler(resources): Promise { + async handler(resources): Promise { const { context, params } = resources; const { dataStream } = params.path; const coreContext = await context.core; diff --git a/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts b/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts index 55233db91565b..fb2edfcc56efd 100644 --- a/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts +++ b/x-pack/test/dataset_quality_api_integration/tests/data_streams/degraded_fields.spec.ts @@ -88,7 +88,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { it('returns no results when dataStream does not have any degraded fields', async () => { const resp = await callApiAs('datasetQualityLogsUser', `${type}-${dataset}-${namespace}`); - expect(resp.body.length).to.be(0); + expect(resp.body.degradedFields.length).to.be(0); }); it('returns results when dataStream do have degraded fields', async () => { @@ -97,9 +97,9 @@ export default function ApiTest({ getService }: FtrProviderContext) { 'datasetQualityLogsUser', `${type}-${degradedFieldDataset}-${namespace}` ); - const degradedFields = resp.body.map((field: DegradedField) => field.fieldName); + const degradedFields = resp.body.degradedFields.map((field: DegradedField) => field.name); - expect(resp.body.length).to.be(2); + expect(resp.body.degradedFields.length).to.be(2); expect(degradedFields).to.eql(expectedDegradedFields); }); }); From 10abcbca248837911ca378e71323928ee6cb7103 Mon Sep 17 00:00:00 2001 From: achyutjhunjhunwala Date: Wed, 29 May 2024 12:45:30 +0200 Subject: [PATCH 11/11] Change tooltip message for degraded fields --- .../dataset_quality/common/translations.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts index 47d1aff3183d7..c57864e8aff2f 100644 --- a/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts +++ b/x-pack/plugins/observability_solution/dataset_quality/common/translations.ts @@ -147,7 +147,8 @@ export const flyoutImprovementText = i18n.translate( export const flyoutImprovementTooltip = i18n.translate( 'xpack.datasetQuality.flyoutDegradedFieldsSectionTooltip', { - defaultMessage: 'Set of degraded fields in the dataset.', + defaultMessage: + 'Set of degraded fields in the dataset. Please not that this list may not be exhaustive.', } );