[Dataset quality] Extracting totalDocs form degradedDocs request (#19…

…8757)

Relates to elastic/logs-dev#183

## Summary
This PR aims to split out `total_docs` from `degraded_docs` request.
This number is no longer relevant only for degraded docs. This PR is a
preparation step for supporting `failed_docs`.

### 🎥 Demo 

https://github.com/user-attachments/assets/7a826715-64e2-4799-8b54-934698df56e2

#### When no documents are found in the selected timerange

https://github.com/user-attachments/assets/de974125-cf45-42d3-932f-32e43b282eb2

#### Filtering datasets

https://github.com/user-attachments/assets/398fc7db-1e38-4998-9ecb-10e8644f812d

### TODO

- [ ] Test in MKI before merging

x-pack/plugins/observability_solution/dataset_quality/common/api_types.ts

@@ -37,6 +37,25 @@ export const dataStreamStatRt = rt.intersection([
 
 export type DataStreamStat = rt.TypeOf<typeof dataStreamStatRt>;
 
+export const dataStreamDocsStatRt = rt.type({
+  dataset: rt.string,
+  count: rt.number,
+});
+
+export type DataStreamDocsStat = rt.TypeOf<typeof dataStreamDocsStatRt>;
+
+export const getDataStreamTotalDocsResponseRt = rt.type({
+  totalDocs: rt.array(dataStreamDocsStatRt),
+});
+
+export type DataStreamTotalDocsResponse = rt.TypeOf<typeof getDataStreamTotalDocsResponseRt>;
+
+export const getDataStreamDegradedDocsResponseRt = rt.type({
+  degradedDocs: rt.array(dataStreamDocsStatRt),
+});
+
+export type DataStreamDegradedDocsResponse = rt.TypeOf<typeof getDataStreamDegradedDocsResponseRt>;
+
 export const integrationDashboardRT = rt.type({
   id: rt.string,
   title: rt.string,
@@ -84,15 +103,6 @@ export const getIntegrationsResponseRt = rt.exact(
 
 export type IntegrationResponse = rt.TypeOf<typeof getIntegrationsResponseRt>;
 
-export const degradedDocsRt = rt.type({
-  dataset: rt.string,
-  count: rt.number,
-  docsCount: rt.number,
-  percentage: rt.number,
-});
-
-export type DegradedDocs = rt.TypeOf<typeof degradedDocsRt>;
-
 export const degradedFieldRt = rt.type({
   name: rt.string,
   count: rt.number,
@@ -188,12 +198,6 @@ export const getDataStreamsStatsResponseRt = rt.exact(
   })
 );
 
-export const getDataStreamsDegradedDocsStatsResponseRt = rt.exact(
-  rt.type({
-    degradedDocs: rt.array(degradedDocsRt),
-  })
-);
-
 export const getDataStreamsSettingsResponseRt = rt.exact(dataStreamSettingsRt);
 
 export const getDataStreamsDetailsResponseRt = rt.exact(dataStreamDetailsRt);

x-pack/plugins/observability_solution/dataset_quality/common/constants.ts

@@ -11,6 +11,7 @@ export const DATASET_QUALITY_APP_ID = 'dataset_quality';
 export const DEFAULT_DATASET_TYPE: DataStreamType = 'logs';
 export const DEFAULT_LOGS_DATA_VIEW = 'logs-*-*';
 
+export const DEFAULT_DATASET_QUALITY: QualityIndicators = 'good';
 export const POOR_QUALITY_MINIMUM_PERCENTAGE = 3;
 export const DEGRADED_QUALITY_MINIMUM_PERCENTAGE = 0;
 
@@ -26,10 +27,8 @@ export const DEFAULT_TIME_RANGE = { from: 'now-24h', to: 'now' };
 export const DEFAULT_DATEPICKER_REFRESH = { value: 60000, pause: false };
 
 export const DEFAULT_DEGRADED_DOCS = {
-  percentage: 0,
   count: 0,
-  docsCount: 0,
-  quality: 'good' as QualityIndicators,
+  percentage: 0,
 };
 
 export const NUMBER_FORMAT = '0,0.[000]';

x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/data_stream_stat.ts

@@ -5,11 +5,11 @@
  * 2.0.
  */
 
-import { DEFAULT_DEGRADED_DOCS } from '../constants';
+import { DataStreamDocsStat } from '../api_types';
+import { DEFAULT_DATASET_QUALITY, DEFAULT_DEGRADED_DOCS } from '../constants';
 import { DataStreamType, QualityIndicators } from '../types';
 import { indexNameToDataStreamParts, mapPercentageToQuality } from '../utils';
 import { Integration } from './integration';
-import { DegradedDocsStat } from './malformed_docs_stat';
 import { DataStreamStatType } from './types';
 
 export class DataStreamStat {
@@ -24,11 +24,11 @@ export class DataStreamStat {
   userPrivileges?: DataStreamStatType['userPrivileges'];
   totalDocs?: DataStreamStatType['totalDocs']; // total datastream docs count
   integration?: Integration;
+  quality: QualityIndicators;
+  docsInTimeRange?: number;
   degradedDocs: {
     percentage: number;
     count: number;
-    docsCount: number; // docs count in the filtered time range
-    quality: QualityIndicators;
   };
 
   private constructor(dataStreamStat: DataStreamStat) {
@@ -43,12 +43,9 @@ export class DataStreamStat {
     this.userPrivileges = dataStreamStat.userPrivileges;
     this.totalDocs = dataStreamStat.totalDocs;
     this.integration = dataStreamStat.integration;
-    this.degradedDocs = {
-      percentage: dataStreamStat.degradedDocs.percentage,
-      count: dataStreamStat.degradedDocs.count,
-      docsCount: dataStreamStat.degradedDocs.docsCount,
-      quality: dataStreamStat.degradedDocs.quality,
-    };
+    this.quality = dataStreamStat.quality;
+    this.docsInTimeRange = dataStreamStat.docsInTimeRange;
+    this.degradedDocs = dataStreamStat.degradedDocs;
   }
 
   public static create(dataStreamStat: DataStreamStatType) {
@@ -65,6 +62,7 @@ export class DataStreamStat {
       lastActivity: dataStreamStat.lastActivity,
       userPrivileges: dataStreamStat.userPrivileges,
       totalDocs: dataStreamStat.totalDocs,
+      quality: DEFAULT_DATASET_QUALITY,
       degradedDocs: DEFAULT_DEGRADED_DOCS,
     };
 
@@ -74,9 +72,11 @@ export class DataStreamStat {
   public static fromDegradedDocStat({
     degradedDocStat,
     datasetIntegrationMap,
+    totalDocs,
   }: {
-    degradedDocStat: DegradedDocsStat;
+    degradedDocStat: DataStreamDocsStat & { percentage: number };
     datasetIntegrationMap: Record<string, { integration: Integration; title: string }>;
+    totalDocs: number;
   }) {
     const { type, dataset, namespace } = indexNameToDataStreamParts(degradedDocStat.dataset);
 
@@ -87,19 +87,23 @@ export class DataStreamStat {
       title: datasetIntegrationMap[dataset]?.title || dataset,
       namespace,
       integration: datasetIntegrationMap[dataset]?.integration,
+      quality: mapPercentageToQuality(degradedDocStat.percentage),
+      docsInTimeRange: totalDocs,
       degradedDocs: {
         percentage: degradedDocStat.percentage,
         count: degradedDocStat.count,
-        docsCount: degradedDocStat.docsCount,
-        quality: mapPercentageToQuality(degradedDocStat.percentage),
       },
     };
 
     return new DataStreamStat(dataStreamStatProps);
   }
 
-  public static calculateFilteredSize({ sizeBytes, totalDocs, degradedDocs }: DataStreamStat) {
+  public static calculateFilteredSize({ sizeBytes, totalDocs, docsInTimeRange }: DataStreamStat) {
     const avgDocSize = sizeBytes && totalDocs ? sizeBytes / totalDocs : 0;
-    return avgDocSize * degradedDocs.docsCount;
+    return avgDocSize * (docsInTimeRange ?? 0);
+  }
+
+  public static calculatePercentage({ totalDocs, count }: { totalDocs?: number; count?: number }) {
+    return totalDocs && count ? (count / totalDocs) * 100 : 0;
   }
 }

x-pack/plugins/observability_solution/dataset_quality/common/data_streams_stats/types.ts

@@ -18,10 +18,14 @@ export type DataStreamStatServiceResponse = GetDataStreamsStatsResponse;
 export type GetDataStreamsDegradedDocsStatsParams =
   APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/degraded_docs`>['params'];
 export type GetDataStreamsDegradedDocsStatsQuery = GetDataStreamsDegradedDocsStatsParams['query'];
-export type GetDataStreamsDegradedDocsStatsResponse =
-  APIReturnType<`GET /internal/dataset_quality/data_streams/degraded_docs`>;
-export type DegradedDocsStatType = GetDataStreamsDegradedDocsStatsResponse['degradedDocs'][0];
-export type DataStreamDegradedDocsStatServiceResponse = DegradedDocsStatType[];
+
+/*
+Types for stats based in documents inside a DataStream
+*/
+
+export type GetDataStreamsTotalDocsParams =
+  APIClientRequestParamsOf<`GET /internal/dataset_quality/data_streams/total_docs`>['params'];
+export type GetDataStreamsTotalDocsQuery = GetDataStreamsTotalDocsParams['query'];
 
 /*
 Types for Degraded Fields inside a DataStream

x-pack/plugins/observability_solution/dataset_quality/public/components/quality_indicator/dataset_quality_indicator.tsx

@@ -19,9 +19,7 @@ export const DatasetQualityIndicator = ({
   isLoading: boolean;
   dataStreamStat: DataStreamStat;
 }) => {
-  const {
-    degradedDocs: { quality },
-  } = dataStreamStat;
+  const { quality } = dataStreamStat;
 
   const translatedQuality = i18n.translate('xpack.datasetQuality.datasetQualityIdicator', {
     defaultMessage: '{quality}',

x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_filters.ts

@@ -49,7 +49,7 @@ export const useDatasetQualityFilters = () => {
       datasets.reduce(
         (acc: Filters, dataset) => ({
           namespaces: [...new Set([...acc.namespaces, dataset.namespace])],
-          qualities: [...new Set([...acc.qualities, dataset.degradedDocs.quality])],
+          qualities: [...new Set([...acc.qualities, dataset.quality])],
           filteredIntegrations: [
             ...new Set([...acc.filteredIntegrations, dataset.integration?.name ?? 'none']),
           ],

x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_quality_table.tsx

@@ -132,8 +132,7 @@ export const useDatasetQualityTable = () => {
       const passesNamespaceFilter =
         namespaces.length === 0 || namespaces.includes(dataset.namespace);
 
-      const passesQualityFilter =
-        qualities.length === 0 || qualities.includes(dataset.degradedDocs.quality);
+      const passesQualityFilter = qualities.length === 0 || qualities.includes(dataset.quality);
 
       const passesQueryFilter = !query || dataset.rawName.includes(query);
 

x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_dataset_telemetry.ts

@@ -77,7 +77,7 @@ function getDatasetEbtProps(
       namespace: dataset.namespace,
       type: dataset.type,
     },
-    data_stream_health: dataset.degradedDocs.quality,
+    data_stream_health: dataset.quality,
     data_stream_aggregatable: nonAggregatableDatasets.some(
       (indexName) => indexName === dataset.rawName
     ),

x-pack/plugins/observability_solution/dataset_quality/public/hooks/use_summary_panel.ts

@@ -84,7 +84,7 @@ const useSummaryPanel = () => {
     datasetsActivity,
 
     numberOfDatasets: filteredItems.length,
-    numberOfDocuments: filteredItems.reduce((acc, curr) => acc + curr.degradedDocs.docsCount, 0),
+    numberOfDocuments: filteredItems.reduce((acc, curr) => acc + curr.docsInTimeRange!, 0),
   };
 };
 

x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/data_streams_stats_client.ts

@@ -10,8 +10,11 @@ import { decodeOrThrow } from '@kbn/io-ts-utils';
 import rison from '@kbn/rison';
 import { KNOWN_TYPES } from '../../../common/constants';
 import {
-  getDataStreamsDegradedDocsStatsResponseRt,
+  DataStreamDegradedDocsResponse,
+  DataStreamTotalDocsResponse,
+  getDataStreamDegradedDocsResponseRt,
   getDataStreamsStatsResponseRt,
+  getDataStreamTotalDocsResponseRt,
   getIntegrationsResponseRt,
   getNonAggregatableDatasetsRt,
   IntegrationResponse,
@@ -20,9 +23,9 @@ import {
 import {
   DataStreamStatServiceResponse,
   GetDataStreamsDegradedDocsStatsQuery,
-  GetDataStreamsDegradedDocsStatsResponse,
   GetDataStreamsStatsQuery,
   GetDataStreamsStatsResponse,
+  GetDataStreamsTotalDocsQuery,
   GetNonAggregatableDataStreamsParams,
 } from '../../../common/data_streams_stats';
 import { Integration } from '../../../common/data_streams_stats/integration';
@@ -56,16 +59,37 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient {
     return { dataStreamsStats, datasetUserPrivileges };
   }
 
+  public async getDataStreamsTotalDocs(params: GetDataStreamsTotalDocsQuery) {
+    const response = await this.http
+      .get<DataStreamTotalDocsResponse>('/internal/dataset_quality/data_streams/total_docs', {
+        query: {
+          ...params,
+        },
+      })
+      .catch((error) => {
+        throw new DatasetQualityError(`Failed to fetch data streams total docs: ${error}`, error);
+      });
+
+    const { totalDocs } = decodeOrThrow(
+      getDataStreamTotalDocsResponseRt,
+      (message: string) =>
+        new DatasetQualityError(
+          `Failed to decode data streams total docs stats response: ${message}`
+        )
+    )(response);
+
+    return totalDocs;
+  }
+
   public async getDataStreamsDegradedStats(params: GetDataStreamsDegradedDocsStatsQuery) {
+    const types = params.types.length === 0 ? KNOWN_TYPES : params.types;
     const response = await this.http
-      .get<GetDataStreamsDegradedDocsStatsResponse>(
-        '/internal/dataset_quality/data_streams/degraded_docs',
-        {
-          query: {
-            ...params,
-          },
-        }
-      )
+      .get<DataStreamDegradedDocsResponse>('/internal/dataset_quality/data_streams/degraded_docs', {
+        query: {
+          ...params,
+          types: rison.encodeArray(types),
+        },
+      })
       .catch((error) => {
         throw new DatasetQualityError(
           `Failed to fetch data streams degraded stats: ${error}`,
@@ -74,7 +98,7 @@ export class DataStreamsStatsClient implements IDataStreamsStatsClient {
       });
 
     const { degradedDocs } = decodeOrThrow(
-      getDataStreamsDegradedDocsStatsResponseRt,
+      getDataStreamDegradedDocsResponseRt,
       (message: string) =>
         new DatasetQualityError(
           `Failed to decode data streams degraded docs stats response: ${message}`

x-pack/plugins/observability_solution/dataset_quality/public/services/data_streams_stats/types.ts

@@ -7,14 +7,14 @@
 
 import { HttpStart } from '@kbn/core/public';
 import {
-  DataStreamDegradedDocsStatServiceResponse,
   DataStreamStatServiceResponse,
   GetDataStreamsDegradedDocsStatsQuery,
   GetDataStreamsStatsQuery,
+  GetDataStreamsTotalDocsQuery,
   GetNonAggregatableDataStreamsParams,
 } from '../../../common/data_streams_stats';
 import { Integration } from '../../../common/data_streams_stats/integration';
-import { NonAggregatableDatasets } from '../../../common/api_types';
+import { DataStreamDocsStat, NonAggregatableDatasets } from '../../../common/api_types';
 
 export type DataStreamsStatsServiceSetup = void;
 
@@ -30,7 +30,8 @@ export interface IDataStreamsStatsClient {
   getDataStreamsStats(params?: GetDataStreamsStatsQuery): Promise<DataStreamStatServiceResponse>;
   getDataStreamsDegradedStats(
     params?: GetDataStreamsDegradedDocsStatsQuery
-  ): Promise<DataStreamDegradedDocsStatServiceResponse>;
+  ): Promise<DataStreamDocsStat[]>;
+  getDataStreamsTotalDocs(params: GetDataStreamsTotalDocsQuery): Promise<DataStreamDocsStat[]>;
   getIntegrations(): Promise<Integration[]>;
   getNonAggregatableDatasets(
     params: GetNonAggregatableDataStreamsParams

x-pack/plugins/observability_solution/dataset_quality/public/state_machines/dataset_quality_controller/src/defaults.ts

@@ -37,7 +37,8 @@ export const DEFAULT_CONTEXT: DefaultDatasetQualityControllerState = {
     canViewIntegrations: true,
   },
   dataStreamStats: [],
-  degradedDocStats: DEFAULT_DICTIONARY_TYPE,
+  degradedDocStats: [],
+  totalDocsStats: DEFAULT_DICTIONARY_TYPE,
   filters: {
     inactive: true,
     fullNames: false,