[8.x] [Cloud Security] Host Name Misconfiguration Datagrid & Refactor CSP Plugin PHASE 2 (#192535)

x-pack/packages/kbn-cloud-security-posture-common/index.ts

@@ -17,7 +17,7 @@ export type {
   BaseCspSetupStatus,
   CspSetupStatus,
 } from './types/status';
-export type { CspFinding } from './types/findings';
+export type { CspFinding, CspFindingResult } from './types/findings';
 export type { BenchmarksCisId } from './types/benchmark';
 export * from './constants';
 export {

x-pack/packages/kbn-cloud-security-posture-common/types/findings.ts

@@ -41,7 +41,7 @@ interface CspFindingCloud {
   region?: string;
 }
 
-interface CspFindingResult {
+export interface CspFindingResult {
   evaluation: 'passed' | 'failed';
   expected?: Record<string, unknown>;
   evidence: Record<string, unknown>;

x-pack/packages/kbn-cloud-security-posture/src/hooks/use_misconfiguration_findings.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { useQuery } from '@tanstack/react-query';
+import { lastValueFrom } from 'rxjs';
+import { CspFinding } from '@kbn/cloud-security-posture-common';
+import { useKibana } from '@kbn/kibana-react-plugin/public';
+import type { CoreStart } from '@kbn/core/public';
+import { showErrorToast } from '../..';
+import type {
+  CspClientPluginStartDeps,
+  LatestFindingsRequest,
+  LatestFindingsResponse,
+  UseMisconfigurationOptions,
+} from '../../type';
+
+import { useGetCspBenchmarkRulesStatesApi } from './use_get_benchmark_rules_state_api';
+import {
+  buildMisconfigurationsFindingsQuery,
+  getMisconfigurationAggregationCount,
+} from '../utils/hooks_utils';
+
+export const useMisconfigurationFindings = (options: UseMisconfigurationOptions) => {
+  const {
+    data,
+    notifications: { toasts },
+  } = useKibana<CoreStart & CspClientPluginStartDeps>().services;
+  const { data: rulesStates } = useGetCspBenchmarkRulesStatesApi();
+
+  return useQuery(
+    ['csp_misconfiguration_findings', { params: options }, rulesStates],
+    async () => {
+      const {
+        rawResponse: { hits, aggregations },
+      } = await lastValueFrom(
+        data.search.search<LatestFindingsRequest, LatestFindingsResponse>({
+          params: buildMisconfigurationsFindingsQuery(options, rulesStates!),
+        })
+      );
+      if (!aggregations) throw new Error('expected aggregations to be defined');
+
+      return {
+        count: getMisconfigurationAggregationCount(aggregations.count.buckets),
+        rows: hits.hits.map((finding) => ({
+          result: finding._source?.result,
+          rule: finding?._source?.rule,
+          resource: finding?._source?.resource,
+        })) as Array<Pick<CspFinding, 'result' | 'rule' | 'resource'>>,
+      };
+    },
+    {
+      enabled: options.enabled && !!rulesStates,
+      keepPreviousData: true,
+      onError: (err: Error) => showErrorToast(toasts, err),
+    }
+  );
+};

x-pack/packages/kbn-cloud-security-posture/src/hooks/use_misconfiguration_preview.ts

@@ -6,118 +6,22 @@
  */
 import { useQuery } from '@tanstack/react-query';
 import { lastValueFrom } from 'rxjs';
-import type { IKibanaSearchResponse, IKibanaSearchRequest } from '@kbn/search-types';
-import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
-import {
-  CDR_MISCONFIGURATIONS_INDEX_PATTERN,
-  LATEST_FINDINGS_RETENTION_POLICY,
-  CspFinding,
-} from '@kbn/cloud-security-posture-common';
-import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
-import { buildMutedRulesFilter } from '@kbn/cloud-security-posture-common';
 import { useKibana } from '@kbn/kibana-react-plugin/public';
 import type { CoreStart } from '@kbn/core/public';
 import { showErrorToast } from '../..';
-import type { CspClientPluginStartDeps } from '../../type';
+import type {
+  CspClientPluginStartDeps,
+  LatestFindingsRequest,
+  LatestFindingsResponse,
+  UseMisconfigurationOptions,
+} from '../../type';
 import { useGetCspBenchmarkRulesStatesApi } from './use_get_benchmark_rules_state_api';
+import {
+  buildMisconfigurationsFindingsQuery,
+  getMisconfigurationAggregationCount,
+} from '../utils/hooks_utils';
 
-interface MisconfigurationPreviewBaseEsQuery {
-  query?: {
-    bool: {
-      filter: estypes.QueryDslQueryContainer[];
-    };
-  };
-}
-
-interface UseMisconfigurationPreviewOptions extends MisconfigurationPreviewBaseEsQuery {
-  sort: string[][];
-  enabled: boolean;
-  pageSize: number;
-}
-
-type LatestFindingsRequest = IKibanaSearchRequest<estypes.SearchRequest>;
-type LatestFindingsResponse = IKibanaSearchResponse<
-  estypes.SearchResponse<CspFinding, FindingsAggs>
->;
-
-interface FindingsAggs {
-  count: estypes.AggregationsMultiBucketAggregateBase<estypes.AggregationsStringRareTermsBucketKeys>;
-}
-
-const RESULT_EVALUATION = {
-  PASSED: 'passed',
-  FAILED: 'failed',
-  UNKNOWN: 'unknown',
-};
-
-export const getFindingsCountAggQueryMisconfigurationPreview = () => ({
-  count: {
-    filters: {
-      other_bucket_key: RESULT_EVALUATION.UNKNOWN,
-      filters: {
-        [RESULT_EVALUATION.PASSED]: { match: { 'result.evaluation': RESULT_EVALUATION.PASSED } },
-        [RESULT_EVALUATION.FAILED]: { match: { 'result.evaluation': RESULT_EVALUATION.FAILED } },
-      },
-    },
-  },
-});
-
-export const getMisconfigurationAggregationCount = (
-  buckets: estypes.AggregationsBuckets<estypes.AggregationsStringRareTermsBucketKeys>
-) => {
-  return Object.entries(buckets).reduce(
-    (evaluation, [key, value]) => {
-      evaluation[key] = (evaluation[key] || 0) + (value.doc_count || 0);
-      return evaluation;
-    },
-    {
-      [RESULT_EVALUATION.PASSED]: 0,
-      [RESULT_EVALUATION.FAILED]: 0,
-      [RESULT_EVALUATION.UNKNOWN]: 0,
-    }
-  );
-};
-
-export const buildMisconfigurationsFindingsQuery = (
-  { query }: UseMisconfigurationPreviewOptions,
-  rulesStates: CspBenchmarkRulesStates
-) => {
-  const mutedRulesFilterQuery = buildMutedRulesFilter(rulesStates);
-
-  return {
-    index: CDR_MISCONFIGURATIONS_INDEX_PATTERN,
-    size: 0,
-    aggs: getFindingsCountAggQueryMisconfigurationPreview(),
-    ignore_unavailable: false,
-    query: buildMisconfigurationsFindingsQueryWithFilters(query, mutedRulesFilterQuery),
-  };
-};
-
-const buildMisconfigurationsFindingsQueryWithFilters = (
-  query: UseMisconfigurationPreviewOptions['query'],
-  mutedRulesFilterQuery: estypes.QueryDslQueryContainer[]
-) => {
-  return {
-    ...query,
-    bool: {
-      ...query?.bool,
-      filter: [
-        ...(query?.bool?.filter ?? []),
-        {
-          range: {
-            '@timestamp': {
-              gte: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
-              lte: 'now',
-            },
-          },
-        },
-      ],
-      must_not: [...mutedRulesFilterQuery],
-    },
-  };
-};
-
-export const useMisconfigurationPreview = (options: UseMisconfigurationPreviewOptions) => {
+export const useMisconfigurationPreview = (options: UseMisconfigurationOptions) => {
   const {
     data,
     notifications: { toasts },
@@ -134,10 +38,10 @@ export const useMisconfigurationPreview = (options: UseMisconfigurationPreviewOp
           params: buildMisconfigurationsFindingsQuery(options, rulesStates!),
         })
       );
-      if (!aggregations) throw new Error('expected aggregations to be defined');
-
+      if (!aggregations && !options.ignore_unavailable)
+        throw new Error('expected aggregations to be defined');
       return {
-        count: getMisconfigurationAggregationCount(aggregations.count.buckets),
+        count: getMisconfigurationAggregationCount(aggregations?.count?.buckets),
       };
     },
     {

x-pack/packages/kbn-cloud-security-posture/src/utils/hooks_utils.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import {
+  CDR_MISCONFIGURATIONS_INDEX_PATTERN,
+  LATEST_FINDINGS_RETENTION_POLICY,
+} from '@kbn/cloud-security-posture-common';
+import type { CspBenchmarkRulesStates } from '@kbn/cloud-security-posture-common/schema/rules/latest';
+import { buildMutedRulesFilter } from '@kbn/cloud-security-posture-common';
+import type { UseMisconfigurationOptions } from '../../type';
+
+const MISCONFIGURATIONS_SOURCE_FIELDS = ['result.*', 'rule.*', 'resource.*'];
+interface AggregationBucket {
+  doc_count?: number;
+}
+
+type AggregationBuckets = Record<string, AggregationBucket>;
+
+const RESULT_EVALUATION = {
+  PASSED: 'passed',
+  FAILED: 'failed',
+  UNKNOWN: 'unknown',
+};
+
+export const getFindingsCountAggQueryMisconfiguration = () => ({
+  count: {
+    filters: {
+      other_bucket_key: RESULT_EVALUATION.UNKNOWN,
+      filters: {
+        [RESULT_EVALUATION.PASSED]: { match: { 'result.evaluation': RESULT_EVALUATION.PASSED } },
+        [RESULT_EVALUATION.FAILED]: { match: { 'result.evaluation': RESULT_EVALUATION.FAILED } },
+      },
+    },
+  },
+});
+
+export const getMisconfigurationAggregationCount = (
+  buckets?: estypes.AggregationsBuckets<estypes.AggregationsStringRareTermsBucketKeys>
+) => {
+  const defaultBuckets: AggregationBuckets = {
+    [RESULT_EVALUATION.PASSED]: { doc_count: 0 },
+    [RESULT_EVALUATION.FAILED]: { doc_count: 0 },
+    [RESULT_EVALUATION.UNKNOWN]: { doc_count: 0 },
+  };
+
+  // if buckets are undefined we will use default buckets
+  const usedBuckets = buckets || defaultBuckets;
+  return Object.entries(usedBuckets).reduce(
+    (evaluation, [key, value]) => {
+      evaluation[key] = (evaluation[key] || 0) + (value.doc_count || 0);
+      return evaluation;
+    },
+    {
+      [RESULT_EVALUATION.PASSED]: 0,
+      [RESULT_EVALUATION.FAILED]: 0,
+      [RESULT_EVALUATION.UNKNOWN]: 0,
+    }
+  );
+};
+
+export const buildMisconfigurationsFindingsQuery = (
+  { query }: UseMisconfigurationOptions,
+  rulesStates: CspBenchmarkRulesStates,
+  isPreview = false
+) => {
+  const mutedRulesFilterQuery = buildMutedRulesFilter(rulesStates);
+
+  return {
+    index: CDR_MISCONFIGURATIONS_INDEX_PATTERN,
+    size: isPreview ? 0 : 500,
+    aggs: getFindingsCountAggQueryMisconfiguration(),
+    ignore_unavailable: true,
+    query: buildMisconfigurationsFindingsQueryWithFilters(query, mutedRulesFilterQuery),
+    _source: MISCONFIGURATIONS_SOURCE_FIELDS,
+  };
+};
+
+const buildMisconfigurationsFindingsQueryWithFilters = (
+  query: UseMisconfigurationOptions['query'],
+  mutedRulesFilterQuery: estypes.QueryDslQueryContainer[]
+) => {
+  return {
+    ...query,
+    bool: {
+      ...query?.bool,
+      filter: [
+        ...(query?.bool?.filter ?? []),
+        {
+          range: {
+            '@timestamp': {
+              gte: `now-${LATEST_FINDINGS_RETENTION_POLICY}`,
+              lte: 'now',
+            },
+          },
+        },
+      ],
+      must_not: [...mutedRulesFilterQuery],
+    },
+  };
+};

x-pack/packages/kbn-cloud-security-posture/type.ts

@@ -22,6 +22,9 @@ import type { FleetStart } from '@kbn/fleet-plugin/public';
 import type { UsageCollectionStart } from '@kbn/usage-collection-plugin/public';
 import { SharePluginStart } from '@kbn/share-plugin/public';
 import { SpacesPluginStart } from '@kbn/spaces-plugin/public';
+import { CspFinding } from '@kbn/cloud-security-posture-common';
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import type { IKibanaSearchResponse, IKibanaSearchRequest } from '@kbn/search-types';
 
 import type { BoolQuery } from '@kbn/es-query';
 export interface FindingsBaseEsQuery {
@@ -51,3 +54,27 @@ export interface CspClientPluginStartDeps {
   // optional
   usageCollection?: UsageCollectionStart;
 }
+
+export interface MisconfigurationBaseEsQuery {
+  query?: {
+    bool: {
+      filter: estypes.QueryDslQueryContainer[];
+    };
+  };
+}
+
+export interface UseMisconfigurationOptions extends MisconfigurationBaseEsQuery {
+  sort: string[][];
+  enabled: boolean;
+  pageSize: number;
+  ignore_unavailable?: boolean;
+}
+
+export type LatestFindingsRequest = IKibanaSearchRequest<estypes.SearchRequest>;
+export type LatestFindingsResponse = IKibanaSearchResponse<
+  estypes.SearchResponse<CspFinding, FindingsAggs>
+>;
+
+export interface FindingsAggs {
+  count: estypes.AggregationsMultiBucketAggregateBase<estypes.AggregationsStringRareTermsBucketKeys>;
+}