From 4b03bee10ee9fc88c2228a4d75e573b643c7c58a Mon Sep 17 00:00:00 2001
From: "opensearch-trigger-bot[bot]"
 <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com>
Date: Thu, 5 Oct 2023 17:57:41 -0700
Subject: [PATCH] Enhance log type filters correlations (#745) (#747)

* updated log type filters; fixed search field

Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com>

* updated log type filters; fixed search field

Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com>

* fixed typo

Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com>

---------

Signed-off-by: Amardeepsingh Siglani <amardeep7194@gmail.com>
(cherry picked from commit b581e5833a773630dd4b58cda00ee419b1b9ca99)

Co-authored-by: Amardeepsingh Siglani <amardeep7194@gmail.com>
---
 .../Correlations/components/FilterGroup.tsx   | 93 ++++++++++++++++---
 .../containers/CorrelationsContainer.tsx      |  2 +
 public/pages/Correlations/utils/constants.tsx | 39 ++++++--
 .../pages/LogTypes/components/LogTypeForm.tsx |  4 +-
 public/pages/LogTypes/utils/helpers.tsx       | 10 +-
 public/store/CorrelationsStore.ts             |  3 -
 public/store/LogTypeStore.ts                  | 16 +++-
 public/utils/{constants.tsx => constants.ts}  | 18 +---
 public/utils/helpers.tsx                      | 60 +++++++-----
 server/services/RuleService.ts                |  6 --
 10 files changed, 171 insertions(+), 80 deletions(-)
 rename public/utils/{constants.tsx => constants.ts} (93%)

diff --git a/public/pages/Correlations/components/FilterGroup.tsx b/public/pages/Correlations/components/FilterGroup.tsx
index b71a15eaa..889a0b8a1 100644
--- a/public/pages/Correlations/components/FilterGroup.tsx
+++ b/public/pages/Correlations/components/FilterGroup.tsx
@@ -12,16 +12,32 @@ import {
   EuiPopoverTitle,
   EuiFieldSearch,
   FilterChecked,
+  EuiPopoverFooter,
+  EuiButtonEmpty,
 } from '@elastic/eui';
 
-export type FilterItem = { name: string | React.ReactNode; id: string; checked?: FilterChecked };
+export type FilterItem = {
+  name: string | React.ReactNode;
+  id: string;
+  visible: boolean;
+  childOptionIds?: Set<string>;
+  checked?: FilterChecked;
+};
 export interface LogTypeFilterGroupProps {
   groupName: string;
   items: FilterItem[];
+  hasGroupOptions?: boolean;
+  hasFooter?: boolean;
   setItems: (items: FilterItem[]) => void;
 }
 
-export const FilterGroup: React.FC<LogTypeFilterGroupProps> = ({ groupName, items, setItems }) => {
+export const FilterGroup: React.FC<LogTypeFilterGroupProps> = ({
+  groupName,
+  items,
+  hasGroupOptions,
+  hasFooter,
+  setItems,
+}) => {
   const [isPopoverOpen, setIsPopoverOpen] = useState(false);
   const [showActiveFilters, setShowActiveFilters] = useState(false);
 
@@ -33,7 +49,7 @@ export const FilterGroup: React.FC<LogTypeFilterGroupProps> = ({ groupName, item
     setIsPopoverOpen(false);
   };
 
-  function updateItem(index: number) {
+  function toggleItem(index: number) {
     if (!items[index]) {
       return;
     }
@@ -54,7 +70,35 @@ export const FilterGroup: React.FC<LogTypeFilterGroupProps> = ({ groupName, item
           checked: 'on',
         };
     }
+    const childIds = newItems[index].childOptionIds;
+    if (childIds) {
+      newItems.forEach((item) => {
+        if (childIds.has(item.id)) {
+          item.checked = newItems[index].checked;
+        }
+      });
+    }
+
+    setItems(newItems);
+    setShowActiveFilters(true);
+  }
 
+  function toggleAll(state: 'on' | undefined) {
+    const newItems = items.map((item) => ({
+      ...item,
+      checked: state,
+    }));
+
+    setItems(newItems);
+    setShowActiveFilters(!!state);
+  }
+
+  function search(term: string) {
+    const newItems = [...items];
+    term = term.toLowerCase();
+    items.forEach((item) => {
+      item.visible = item.id.toLowerCase().includes(term);
+    });
     setItems(newItems);
     setShowActiveFilters(true);
   }
@@ -83,20 +127,39 @@ export const FilterGroup: React.FC<LogTypeFilterGroupProps> = ({ groupName, item
         panelPaddingSize="none"
       >
         <EuiPopoverTitle paddingSize="s">
-          <EuiFieldSearch compressed />
+          <EuiFieldSearch compressed onSearch={search} />
         </EuiPopoverTitle>
-        <div className="ouiFilterSelect__items">
-          {items.map((item, index) => (
-            <EuiFilterSelectItem
-              checked={item.checked}
-              key={index}
-              onClick={() => updateItem(index)}
-              showIcons={true}
-            >
-              {item.name}
-            </EuiFilterSelectItem>
-          ))}
+        <div
+          className="ouiFilterSelect__items"
+          style={hasFooter ? { maxHeight: 400, overflow: 'scroll' } : undefined}
+        >
+          {items.map((item, index) => {
+            const itemStyle: any = {};
+            itemStyle['paddingLeft'] =
+              hasGroupOptions && !item.childOptionIds ? 20 : itemStyle['paddingLeft'];
+            itemStyle['display'] = !item.visible ? 'none' : itemStyle['display'];
+
+            return (
+              <EuiFilterSelectItem
+                checked={item.checked}
+                key={index}
+                onClick={() => toggleItem(index)}
+                showIcons={true}
+                style={itemStyle}
+              >
+                {item.name}
+              </EuiFilterSelectItem>
+            );
+          })}
         </div>
+        {hasFooter && (
+          <EuiPopoverFooter>
+            <div>
+              <EuiButtonEmpty onClick={() => toggleAll('on')}>Select all</EuiButtonEmpty>
+              <EuiButtonEmpty onClick={() => toggleAll(undefined)}>Deselect all</EuiButtonEmpty>
+            </div>
+          </EuiPopoverFooter>
+        )}
       </EuiPopover>
     </EuiFilterGroup>
   );
diff --git a/public/pages/Correlations/containers/CorrelationsContainer.tsx b/public/pages/Correlations/containers/CorrelationsContainer.tsx
index 55dcbf747..aa052930c 100644
--- a/public/pages/Correlations/containers/CorrelationsContainer.tsx
+++ b/public/pages/Correlations/containers/CorrelationsContainer.tsx
@@ -527,6 +527,8 @@ export class Correlations extends React.Component<CorrelationsProps, Correlation
                         <FilterGroup
                           groupName="Log types"
                           items={this.state.logTypeFilterOptions}
+                          hasGroupOptions={true}
+                          hasFooter={true}
                           setItems={this.onLogTypeFilterChange}
                         />
                       </EuiFilterGroup>
diff --git a/public/pages/Correlations/utils/constants.tsx b/public/pages/Correlations/utils/constants.tsx
index cb6e1c1b3..8c140ee62 100644
--- a/public/pages/Correlations/utils/constants.tsx
+++ b/public/pages/Correlations/utils/constants.tsx
@@ -7,7 +7,9 @@ import React from 'react';
 import { CorrelationGraphData } from '../../../../types';
 import { ruleSeverity, ruleTypes } from '../../Rules/utils/constants';
 import { FilterItem } from '../components/FilterGroup';
-import { EuiIcon } from '@elastic/eui';
+import { EuiIcon, EuiTitle } from '@elastic/eui';
+import { logTypeCategories, logTypesByCategories } from '../../../utils/constants';
+import _ from 'lodash';
 
 export const graphRenderOptions = {
   nodes: {
@@ -47,12 +49,34 @@ export const graphRenderOptions = {
   },
 };
 
-export const getDefaultLogTypeFilterItemOptions: () => FilterItem[] = () =>
-  Object.values(ruleTypes).map((type) => ({
-    name: `${type.label}`,
-    id: type.label.toLowerCase(),
-    checked: 'on',
-  }));
+export const getDefaultLogTypeFilterItemOptions: () => FilterItem[] = () => {
+  const options: FilterItem[] = [];
+  logTypeCategories.forEach((category) => {
+    const logTypes = logTypesByCategories[category];
+    options.push({
+      name: (
+        <EuiTitle size="xxs">
+          <h4>{category}</h4>
+        </EuiTitle>
+      ),
+      id: category,
+      checked: 'on',
+      childOptionIds: new Set(logTypes.map(({ name }) => name)),
+      visible: true,
+    });
+
+    logTypes.forEach(({ name }) => {
+      options.push({
+        name: _.capitalize(name),
+        id: name,
+        checked: 'on',
+        visible: true,
+      });
+    });
+  });
+
+  return options;
+};
 
 export const defaultSeverityFilterItemOptions: FilterItem[] = Object.values(ruleSeverity).map(
   (sev) => {
@@ -64,6 +88,7 @@ export const defaultSeverityFilterItemOptions: FilterItem[] = Object.values(rule
       ),
       id: sev.value,
       checked: 'on',
+      visible: true,
     };
   }
 );
diff --git a/public/pages/LogTypes/components/LogTypeForm.tsx b/public/pages/LogTypes/components/LogTypeForm.tsx
index c2e302082..9f1cfdc90 100644
--- a/public/pages/LogTypes/components/LogTypeForm.tsx
+++ b/public/pages/LogTypes/components/LogTypeForm.tsx
@@ -20,7 +20,7 @@ import React from 'react';
 import { LOG_TYPE_NAME_REGEX, validateName } from '../../../utils/validation';
 import { NotificationsStart } from 'opensearch-dashboards/public';
 import { useState } from 'react';
-import { logTypeCategoryOptions } from '../../../utils/constants';
+import { getLogTypeCategoryOptions } from '../../../utils/helpers';
 
 export interface LogTypeFormProps {
   logTypeDetails: LogTypeItem;
@@ -119,7 +119,7 @@ export const LogTypeForm: React.FC<LogTypeFormProps> = ({
       <EuiSpacer />
       <EuiFormRow label="Category" isInvalid={!!categoryError} error={categoryError}>
         <EuiSuperSelect
-          options={logTypeCategoryOptions.map((option) => ({
+          options={getLogTypeCategoryOptions().map((option) => ({
             ...option,
             disabled: !isEditMode || (isEditMode && !!logTypeDetails.detectionRulesCount),
           }))}
diff --git a/public/pages/LogTypes/utils/helpers.tsx b/public/pages/LogTypes/utils/helpers.tsx
index 3c09c9803..4826ff4e1 100644
--- a/public/pages/LogTypes/utils/helpers.tsx
+++ b/public/pages/LogTypes/utils/helpers.tsx
@@ -9,7 +9,7 @@ import { LogType } from '../../../../types';
 import { capitalize } from 'lodash';
 import { Search } from '@opensearch-project/oui/src/eui_components/basic_table';
 import { ruleSource } from '../../Rules/utils/constants';
-import { logTypesByCategories } from '../../../utils/constants';
+import { logTypeCategories } from '../../../utils/constants';
 
 export const getLogTypesTableColumns = (
   showDetails: (id: string) => void,
@@ -71,11 +71,9 @@ export const getLogTypesTableSearchConfig = (): Search => {
         field: 'category',
         name: 'Category',
         multiSelect: 'or',
-        options: Object.keys(logTypesByCategories)
-          .map((category) => ({
-            value: category,
-          }))
-          .sort((a, b) => (a.value < b.value ? -1 : a.value > b.value ? 1 : 0)),
+        options: logTypeCategories.map((category) => ({
+          value: category,
+        })),
       },
       {
         type: 'field_value_selection',
diff --git a/public/store/CorrelationsStore.ts b/public/store/CorrelationsStore.ts
index ba2a53240..86d9b3adf 100644
--- a/public/store/CorrelationsStore.ts
+++ b/public/store/CorrelationsStore.ts
@@ -226,9 +226,6 @@ export class CorrelationsStore implements ICorrelationsStore {
         if (findingRes.ok) {
           findingRes.response.findings.forEach((f) => {
             const rule = allRules.find((rule) => rule._id === f.queries[0].id);
-            if (!rule) {
-              console.log(f.queries[0].id);
-            }
             findings[f.id] = {
               ...f,
               id: f.id,
diff --git a/public/store/LogTypeStore.ts b/public/store/LogTypeStore.ts
index 2e49a5725..3528406df 100644
--- a/public/store/LogTypeStore.ts
+++ b/public/store/LogTypeStore.ts
@@ -9,7 +9,7 @@ import LogTypeService from '../services/LogTypeService';
 import { errorNotificationToast } from '../utils/helpers';
 import { DataStore } from './DataStore';
 import { ruleTypes } from '../pages/Rules/utils/constants';
-import { logTypesByCategories } from '../utils/constants';
+import { logTypeCategories, logTypesByCategories } from '../utils/constants';
 
 export class LogTypeStore {
   constructor(private service: LogTypeService, private notifications: NotificationsStart) {}
@@ -73,6 +73,20 @@ export class LogTypeStore {
         logTypesByCategories[logType.category] = logTypesByCategories[logType.category] || [];
         logTypesByCategories[logType.category].push(logType);
       });
+      logTypeCategories.splice(
+        0,
+        logTypeCategories.length,
+        ...Object.keys(logTypesByCategories).sort((a, b) => {
+          if (a === 'Other') {
+            return 1;
+          } else if (b === 'Other') {
+            return -1;
+          } else {
+            return a < b ? -1 : a > b ? 1 : 0;
+          }
+        })
+      );
+
       return logTypes;
     }
 
diff --git a/public/utils/constants.tsx b/public/utils/constants.ts
similarity index 93%
rename from public/utils/constants.tsx
rename to public/utils/constants.ts
index 0acfc360c..4bcf6eddf 100644
--- a/public/utils/constants.tsx
+++ b/public/utils/constants.ts
@@ -3,12 +3,10 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import React from 'react';
 import { SimpleSavedObject } from 'opensearch-dashboards/public';
 import { Detector, LogType, ServerResponse } from '../../types';
 import { DetectorInput, PeriodSchedule } from '../../models/interfaces';
 import { DetectorHit } from '../../server/models/interfaces';
-import { EuiText } from '@elastic/eui';
 import _ from 'lodash';
 
 export const DATE_MATH_FORMAT = 'YYYY-MM-DDTHH:mm:ss.SSSZ';
@@ -177,7 +175,7 @@ export const pendingDashboardCreations: {
   [detectorId: string]: undefined | Promise<void | ServerResponse<SimpleSavedObject<unknown>>>;
 } = {};
 
-const logTypeCategoryInfo = [
+export const logTypeCategoryDescription: { name: string; description: string }[] = [
   { name: 'Access Management', description: 'User access, authentication, group management' },
   { name: 'Applications', description: 'Application lifecycle, API, and web resources activities' },
   { name: 'Cloud Services', description: 'Services managed by cloud providers' },
@@ -186,17 +184,5 @@ const logTypeCategoryInfo = [
   { name: 'Other', description: 'Logs not covered in other categories' },
 ];
 
-export const logTypeCategoryOptions: any[] = logTypeCategoryInfo.map(({ name, description }) => ({
-  value: name,
-  inputDisplay: name,
-  dropdownDisplay: (
-    <>
-      <strong>{name}</strong>
-      <EuiText size="s" color="subdued">
-        <p className="ouiTextColor--subdued">{description}</p>
-      </EuiText>
-    </>
-  ),
-}));
-
+export const logTypeCategories: string[] = [];
 export const logTypesByCategories: { [category: string]: LogType[] } = {};
diff --git a/public/utils/helpers.tsx b/public/utils/helpers.tsx
index c194c3649..ef30e0019 100644
--- a/public/utils/helpers.tsx
+++ b/public/utils/helpers.tsx
@@ -17,7 +17,13 @@ import {
 import moment from 'moment';
 import { PeriodSchedule } from '../../models/interfaces';
 import React from 'react';
-import { DEFAULT_EMPTY_DATA, logTypesByCategories, scheduleUnitText } from './constants';
+import {
+  DEFAULT_EMPTY_DATA,
+  logTypeCategories,
+  logTypeCategoryDescription,
+  logTypesByCategories,
+  scheduleUnitText,
+} from './constants';
 import {
   RuleItem,
   RuleItemInfo,
@@ -164,7 +170,7 @@ export function renderVisualization(spec: TopLevelSpec, containerId: string) {
       console.error(err)
     );
   } catch (error) {
-    console.log(error);
+    console.error(error);
   }
 
   function renderVegaSpec(spec: {}) {
@@ -319,28 +325,19 @@ export const getSeverityBadge = (severity: string) => {
 };
 
 export function formatToLogTypeOptions(logTypesByCategories: { [category: string]: LogType[] }) {
-  return Object.entries(logTypesByCategories)
-    .map(([category, logTypes]) => {
-      return {
-        label: category,
-        value: category,
-        options: logTypes
-          .map(({ name }) => ({
-            label: name,
-            value: name.toLowerCase(),
-          }))
-          .sort((a, b) => (a.label < b.label ? -1 : a.label > b.label ? 1 : 0)),
-      };
-    })
-    .sort((a, b) => {
-      if (a.label === 'Other') {
-        return 1;
-      } else if (b.label === 'Other') {
-        return -1;
-      } else {
-        return a.label < b.label ? -1 : a.label > b.label ? 1 : 0;
-      }
-    });
+  return logTypeCategories.map((category) => {
+    const logTypes = logTypesByCategories[category];
+    return {
+      label: category,
+      value: category,
+      options: logTypes
+        .map(({ name }) => ({
+          label: name,
+          value: name.toLowerCase(),
+        }))
+        .sort((a, b) => (a.label < b.label ? -1 : a.label > b.label ? 1 : 0)),
+    };
+  });
 }
 
 export async function getLogTypeOptions() {
@@ -375,3 +372,18 @@ export function getLogTypeFilterOptions() {
 
   return options;
 }
+
+export function getLogTypeCategoryOptions(): any[] {
+  return logTypeCategoryDescription.map(({ name, description }) => ({
+    value: name,
+    inputDisplay: name,
+    dropdownDisplay: (
+      <>
+        <strong>{name}</strong>
+        <EuiText size="s" color="subdued">
+          <p className="ouiTextColor--subdued">{description}</p>
+        </EuiText>
+      </>
+    ),
+  }));
+}
diff --git a/server/services/RuleService.ts b/server/services/RuleService.ts
index 1d46741d5..bee609eb9 100644
--- a/server/services/RuleService.ts
+++ b/server/services/RuleService.ts
@@ -80,9 +80,7 @@ export default class RulesService {
       if (false_positives.length > 0) {
         jsonPayload['falsepositives'] = false_positives.map((falsePos) => falsePos.value);
       }
-      console.log(jsonPayload);
       const ruleYamlPayload = safeDump(jsonPayload);
-      console.log(ruleYamlPayload);
 
       const params: CreateRuleParams = {
         body: ruleYamlPayload,
@@ -226,12 +224,8 @@ export default class RulesService {
       if (false_positives.length > 0) {
         jsonPayload['falsepositives'] = false_positives.map((falsePos) => falsePos.value);
       }
-      console.log(jsonPayload);
 
       const ruleYamlPayload = safeDump(jsonPayload);
-
-      console.log(ruleYamlPayload);
-
       const params: UpdateRuleParams = { body: ruleYamlPayload, category, ruleId };
       const { callAsCurrentUser: callWithRequest } = this.osDriver.asScoped(request);
       const createRuleResponse: UpdateRuleResponse = await callWithRequest(