[Security Solution][Investigations] - Fix ESQL query size (elastic#17…

…1789) ## Summary This PR improves a potential performance issue for timeline currently. When users load a security solution page with timeline, the default ESQL tab query is run in the background as the query is a part of the url state. While this can be convenient for the speed of retrieval when opening the ESQL tab, it can lead to some performance issues on initial load depending on the number of fields/size of the data in the user's default security indices. To improve the performance of this initial query, we'll limit the query to only initially retrieve the default Timeline columns via the ESQL `keep` clause. `keep @timestamp, message, event.category, event.action, host.names, source.ip, destination.ip, user.name` will be appended to the end of the current default query <img width="1680" alt="image" src="https://github.com/elastic/kibana/assets/17211684/5fc3ae69-8ae6-4143-8f75-7cf69032e63a"> **Regarding the unskipped test:** The tests were flaky due to the fact that interacting with the monaco editor with the cypress `type` commands didn't allow for deleting of the existing text, but this was able to be remedied by first expanding the editor which allows the text to be more easily interacted with via the cypress commands. [Buildkite (x100) test run](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4174#_) - The one failure is due to a flaky test in an unrelated correlation tab. --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
ThomThomson · Dec 6, 2023 · 6349d1c · 6349d1c
1 parent fd30f25
commit 6349d1c
Show file tree

Hide file tree

Showing 8 changed files with 31 additions and 25 deletions.
diff --git a/...lution/public/common/components/discover_in_timeline/use_discover_in_timeline_actions.tsx b/...lution/public/common/components/discover_in_timeline/use_discover_in_timeline_actions.tsx
@@ -14,6 +14,7 @@ import type { SavedSearch } from '@kbn/saved-search-plugin/common';
 import type { DiscoverAppState } from '@kbn/discover-plugin/public/application/main/services/discover_app_state_container';
 import type { TimeRange } from '@kbn/es-query';
 import { useMutation, useQueryClient } from '@tanstack/react-query';
+import { defaultHeaders } from '@kbn/securitysolution-data-table';
 import { timelineDefaults } from '../../../timelines/store/timeline/defaults';
 import { TimelineId } from '../../../../common/types';
 import { timelineActions, timelineSelectors } from '../../../timelines/store/timeline';
@@ -80,10 +81,12 @@ export const useDiscoverInTimelineActions = (
     const localDataViewId = dataViewId ?? 'security-solution-default';
 
     const dataView = await dataViewService.get(localDataViewId);
-
+    const defaultColumns = defaultHeaders.map((header) => header.id);
     return {
       query: {
-        esql: dataView ? `from ${dataView.getIndexPattern()} | limit 10` : '',
+        esql: dataView
+          ? `from ${dataView.getIndexPattern()} | limit 10 | keep ${defaultColumns.join(', ')}`
+          : '',
       },
       sort: [['@timestamp', 'desc']],
       columns: [],

diff --git a/...st/security_solution_cypress/cypress/e2e/investigations/timelines/esql/cell_actions.cy.ts b/...st/security_solution_cypress/cypress/e2e/investigations/timelines/esql/cell_actions.cy.ts
@@ -15,7 +15,7 @@ import { waitForDiscoverGridToLoad } from '../../../../tasks/discover';
 import { updateDateRangeInLocalDatePickers } from '../../../../tasks/date_picker';
 import { login } from '../../../../tasks/login';
 import { visitWithTimeRange } from '../../../../tasks/navigation';
-import { createNewTimeline, gotToEsqlTab } from '../../../../tasks/timeline';
+import { createNewTimeline, goToEsqlTab } from '../../../../tasks/timeline';
 import { ALERTS_URL } from '../../../../urls/navigation';
 
 const INITIAL_START_DATE = 'Jan 18, 2021 @ 20:33:29.186';
@@ -33,7 +33,7 @@ describe.skip(
       login();
       visitWithTimeRange(ALERTS_URL);
       createNewTimeline();
-      gotToEsqlTab();
+      goToEsqlTab();
       updateDateRangeInLocalDatePickers(DISCOVER_CONTAINER, INITIAL_START_DATE, INITIAL_END_DATE);
       waitForDiscoverGridToLoad();
     });

diff --git a/...press/cypress/e2e/investigations/timelines/esql/discover_timeline_state_integration.cy.ts b/...press/cypress/e2e/investigations/timelines/esql/discover_timeline_state_integration.cy.ts
@@ -34,7 +34,7 @@ import {
   addDescriptionToTimeline,
   addNameToTimelineAndSave,
   createNewTimeline,
-  gotToEsqlTab,
+  goToEsqlTab,
   openTimelineById,
   openTimelineFromSettings,
 } from '../../../../tasks/timeline';
@@ -107,14 +107,14 @@ describe.skip(
       login();
       visitWithTimeRange(ALERTS_URL);
       createNewTimeline();
-      gotToEsqlTab();
+      goToEsqlTab();
       updateDateRangeInLocalDatePickers(DISCOVER_CONTAINER, INITIAL_START_DATE, INITIAL_END_DATE);
     });
     context('save/restore', () => {
       it('should be able create an empty timeline with default discover state', () => {
         addNameToTimelineAndSave('Timerange timeline');
         createNewTimeline();
-        gotToEsqlTab();
+        goToEsqlTab();
         cy.get(GET_LOCAL_SHOW_DATES_BUTTON(DISCOVER_CONTAINER)).should(
           'contain.text',
           `Last 15 minutes`
@@ -141,7 +141,7 @@ describe.skip(
             openTimelineFromSettings();
             openTimelineById(timelineId);
             cy.get(LOADING_INDICATOR).should('not.exist');
-            gotToEsqlTab();
+            goToEsqlTab();
             verifyDiscoverEsqlQuery(esqlQuery);
             cy.get(GET_DISCOVER_DATA_GRID_CELL_HEADER(column1)).should('exist');
             cy.get(GET_DISCOVER_DATA_GRID_CELL_HEADER(column2)).should('exist');
@@ -191,7 +191,7 @@ describe.skip(
             openTimelineFromSettings();
             openTimelineById(timelineId);
             cy.get(LOADING_INDICATOR).should('not.exist');
-            gotToEsqlTab();
+            goToEsqlTab();
             cy.get(DISCOVER_DATA_VIEW_SWITCHER.BTN).should('not.exist');
           });
       });

diff --git a/...test/security_solution_cypress/cypress/e2e/investigations/timelines/esql/esql_state.cy.ts b/...test/security_solution_cypress/cypress/e2e/investigations/timelines/esql/esql_state.cy.ts
@@ -20,17 +20,16 @@ import {
 import { updateDateRangeInLocalDatePickers } from '../../../../tasks/date_picker';
 import { login } from '../../../../tasks/login';
 import { visitWithTimeRange } from '../../../../tasks/navigation';
-import { createNewTimeline, gotToEsqlTab, openActiveTimeline } from '../../../../tasks/timeline';
+import { closeTimeline, goToEsqlTab, openActiveTimeline } from '../../../../tasks/timeline';
 import { ALERTS_URL } from '../../../../urls/navigation';
 import { ALERTS, CSP_FINDINGS } from '../../../../screens/security_header';
 
 const INITIAL_START_DATE = 'Jan 18, 2021 @ 20:33:29.186';
 const INITIAL_END_DATE = 'Jan 19, 2024 @ 20:33:29.186';
 const DEFAULT_ESQL_QUERY =
-  'from .alerts-security.alerts-default,apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-* | limit 10';
+  'from .alerts-security.alerts-default,apm-*-transaction*,auditbeat-*,endgame-*,filebeat-*,logs-*,packetbeat-*,traces-apm*,winlogbeat-*,-*elastic-cloud-logs-* | limit 10 | keep @timestamp, message, event.category, event.action, host.name, source.ip, destination.ip, user.name';
 
-// FLAKY: https://github.com/elastic/kibana/issues/169093
-describe.skip(
+describe(
   'Timeline Discover ESQL State',
   {
     tags: ['@ess'],
@@ -39,8 +38,8 @@ describe.skip(
     beforeEach(() => {
       login();
       visitWithTimeRange(ALERTS_URL);
-      createNewTimeline();
-      gotToEsqlTab();
+      openActiveTimeline();
+      goToEsqlTab();
       updateDateRangeInLocalDatePickers(DISCOVER_CONTAINER, INITIAL_START_DATE, INITIAL_END_DATE);
     });
     it('should not allow the dataview to be changed', () => {
@@ -53,20 +52,22 @@ describe.skip(
       const esqlQuery = 'from auditbeat-* | limit 5';
       addDiscoverEsqlQuery(esqlQuery);
       submitDiscoverSearchBar();
+      closeTimeline();
       navigateFromHeaderTo(CSP_FINDINGS);
       navigateFromHeaderTo(ALERTS);
       openActiveTimeline();
-      gotToEsqlTab();
+      goToEsqlTab();
 
       verifyDiscoverEsqlQuery(esqlQuery);
     });
     it('should remember columns when navigating away and back to discover ', () => {
       addFieldToTable('host.name');
       addFieldToTable('user.name');
+      closeTimeline();
       navigateFromHeaderTo(CSP_FINDINGS);
       navigateFromHeaderTo(ALERTS);
       openActiveTimeline();
-      gotToEsqlTab();
+      goToEsqlTab();
       cy.get(GET_DISCOVER_DATA_GRID_CELL_HEADER('host.name')).should('exist');
       cy.get(GET_DISCOVER_DATA_GRID_CELL_HEADER('user.name')).should('exist');
     });

diff --git a/...t/security_solution_cypress/cypress/e2e/investigations/timelines/esql/search_filter.cy.ts b/...t/security_solution_cypress/cypress/e2e/investigations/timelines/esql/search_filter.cy.ts
@@ -23,7 +23,7 @@ import {
   addFieldToTable,
   convertNBSPToSP,
 } from '../../../../tasks/discover';
-import { createNewTimeline, gotToEsqlTab } from '../../../../tasks/timeline';
+import { createNewTimeline, goToEsqlTab } from '../../../../tasks/timeline';
 import { login } from '../../../../tasks/login';
 import { visitWithTimeRange } from '../../../../tasks/navigation';
 import { ALERTS_URL } from '../../../../urls/navigation';
@@ -44,7 +44,7 @@ describe.skip(
       login();
       visitWithTimeRange(ALERTS_URL);
       createNewTimeline();
-      gotToEsqlTab();
+      goToEsqlTab();
       updateDateRangeInLocalDatePickers(DISCOVER_CONTAINER, INITIAL_START_DATE, INITIAL_END_DATE);
     });
 

diff --git a/x-pack/test/security_solution_cypress/cypress/screens/discover.ts b/x-pack/test/security_solution_cypress/cypress/screens/discover.ts
@@ -32,7 +32,8 @@ export const DISCOVER_ESQL_INPUT = `${DISCOVER_CONTAINER} ${getDataTestSubjectSe
 
 export const DISCOVER_ESQL_INPUT_TEXT_CONTAINER = `${DISCOVER_ESQL_INPUT} .view-lines`;
 
-export const DISCOVER_ESQL_EDITABLE_INPUT = `${DISCOVER_ESQL_INPUT} textarea:first`;
+export const DISCOVER_ESQL_INPUT_EXPAND = getDataTestSubjectSelector('TextBasedLangEditor-expand');
+export const DISCOVER_ESQL_EDITABLE_INPUT = `${DISCOVER_ESQL_INPUT} textarea`;
 
 export const DISCOVER_ADD_FILTER = `${DISCOVER_CONTAINER} ${getDataTestSubjectSelector(
   'addFilter'

diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/discover.ts b/x-pack/test/security_solution_cypress/cypress/tasks/discover.ts
@@ -16,9 +16,10 @@ import {
   DISCOVER_DATA_VIEW_EDITOR_FLYOUT,
   DISCOVER_FIELD_LIST_LOADING,
   DISCOVER_ESQL_EDITABLE_INPUT,
+  DISCOVER_ESQL_INPUT_EXPAND,
 } from '../screens/discover';
 import { GET_LOCAL_SEARCH_BAR_SUBMIT_BUTTON } from '../screens/search_bar';
-import { gotToEsqlTab } from './timeline';
+import { goToEsqlTab } from './timeline';
 
 export const switchDataViewTo = (dataviewName: string) => {
   openDataViewSwitcher();
@@ -48,16 +49,16 @@ export const waitForDiscoverGridToLoad = () => {
 export const selectCurrentDiscoverEsqlQuery = (
   discoverEsqlInput = DISCOVER_ESQL_EDITABLE_INPUT
 ) => {
-  gotToEsqlTab();
+  goToEsqlTab();
   cy.get(discoverEsqlInput).should('be.visible').click();
   cy.get(discoverEsqlInput).should('be.focused');
+  cy.get(DISCOVER_ESQL_INPUT_EXPAND).click();
   cy.get(discoverEsqlInput).type(Cypress.platform === 'darwin' ? '{cmd+a}' : '{ctrl+a}');
 };
 
 export const addDiscoverEsqlQuery = (esqlQuery: string) => {
   // ESQL input uses the monaco editor which doesn't allow for traditional input updates
   selectCurrentDiscoverEsqlQuery(DISCOVER_ESQL_EDITABLE_INPUT);
-  cy.get(DISCOVER_ESQL_EDITABLE_INPUT).clear();
   cy.get(DISCOVER_ESQL_EDITABLE_INPUT).type(`${esqlQuery}`);
   cy.get(DISCOVER_ESQL_EDITABLE_INPUT).blur();
   cy.get(GET_LOCAL_SEARCH_BAR_SUBMIT_BUTTON(DISCOVER_CONTAINER)).realClick();

diff --git a/x-pack/test/security_solution_cypress/cypress/tasks/timeline.ts b/x-pack/test/security_solution_cypress/cypress/tasks/timeline.ts
@@ -156,9 +156,9 @@ export const goToNotesTab = (): Cypress.Chainable<JQuery<HTMLElement>> => {
   return cy.get(NOTES_TAB_BUTTON);
 };
 
-export const gotToEsqlTab = () => {
+export const goToEsqlTab = () => {
   recurse(
-    () => cy.get(ESQL_TAB).should('be.visible').click({ force: true }),
+    () => cy.get(ESQL_TAB).should('be.visible').click(),
     ($el) => expect($el).to.have.class('euiTab-isSelected'),
     {
       delay: 500,